Files
pipelines/internal/server/run_reattach_test.go
T
alan aa8eddd686 Keep OpenCode/Codex chat and stage sessions running across server restarts
A stopping server no longer interrupts harness sessions. Runs are canceled
with a shutdown cause, so stage clients leave admitted sessions running and
owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait,
read back the result, continue the graph), restarts unadmitted visits,
continues runs between stages, and falls back to interrupted with a reason
when the harness is gone. Chats with live work are recorded on shutdown and
reattached on start; turns that ended during the downtime are folded in
from history. The server-spawned V2 chat serve is now a detached owned
service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
2026-11-04 16:17:08 +00:00

263 lines
9.8 KiB
Go

package server
import (
"context"
"fmt"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/opencode"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// restartFixture runs a two-stage pipeline on a detaching runner, stops that
// "server" while stage one is in flight, and reopens the store for a second
// runner, the way a restart does.
type restartFixture struct {
path, root, workspace string
runID int64
}
// stopMidStage starts the run and shuts the first runner down while stage one
// is executing. firstStage is the first process's executor for stage one; it
// must block until its context ends.
func stopMidStage(t *testing.T, firstStage func(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error)) restartFixture {
t.Helper()
f := restartFixture{path: filepath.Join(t.TempDir(), "restart.db"), root: t.TempDir(), workspace: t.TempDir()}
s, err := pipeline.Open(f.path)
if err != nil {
t.Fatal(err)
}
ctx, shutdown := context.WithCancel(context.Background())
if _, err := s.Submit(ctx, "restart", pipeline.Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
r, err := newRunnerReattaching(ctx, s, f.root, true)
if err != nil {
t.Fatal(err)
}
r.exportSession = validGateExport
entered := make(chan struct{})
var id atomic.Int64
r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "one" {
t.Errorf("the first process reached stage %s", def.Key)
return stage.Result{}, fmt.Errorf("unexpected stage")
}
for id.Load() == 0 {
time.Sleep(time.Millisecond)
}
close(entered)
return firstStage(ctx, s, id.Load())
}
d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "restart", Workspace: f.workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
id.Store(d.ID)
f.runID = d.ID
<-entered
shutdown() // SIGTERM
r.wg.Wait()
r.schedulerWG.Wait()
persisted, err := s.GetRun(context.Background(), f.runID)
if err != nil {
t.Fatal(err)
}
if persisted.Status != protocol.RunStatusRunning || len(persisted.Steps) != 1 || persisted.Steps[0].Status != protocol.StepStatusRunning {
t.Fatalf("shutdown settled the run instead of leaving it in flight: %+v", persisted)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
return f
}
// restart opens the second process's runner on the same database.
func (f restartFixture) restart(t *testing.T, execute func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error)) (*runner, *pipeline.Store) {
t.Helper()
s, err := pipeline.Open(f.path)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunnerReattaching(ctx, s, f.root, true)
if err != nil {
cancel()
s.Close()
t.Fatal(err)
}
t.Cleanup(func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() })
r.exportSession = validGateExport
r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
return execute(ctx, s, def)
}
return r, s
}
func seamRestartRecovery(t *testing.T, alive error) *atomic.Int32 {
t.Helper()
interrupts := &atomic.Int32{}
previousAlive, previousInterrupt := stageServiceAlive, interruptRecoveredSession
stageServiceAlive = func(context.Context, pipeline.SessionOwner) error { return alive }
interruptRecoveredSession = func(context.Context, pipeline.SessionOwner) error { interrupts.Add(1); return nil }
t.Cleanup(func() { stageServiceAlive, interruptRecoveredSession = previousAlive, previousInterrupt })
return interrupts
}
// ownedLiveStage is a first-process stage one that admitted session
// ses_live (recorded like a real client does) and is still running at shutdown.
func ownedLiveStage(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error) {
if err := s.RunSession(context.Background(), id, 1, "ses_live", true, "v2", "ref-live"); err != nil {
return stage.Result{}, err
}
<-ctx.Done()
if stage.Detached(ctx) {
return stage.Result{}, stage.ErrDetached
}
return stage.Result{}, ctx.Err()
}
// TestStageRunContinuesToNextStageAfterRestart: the stage in flight at a server
// restart is re-adopted (never re-prompted, never interrupted), its result is
// folded in, and the run continues with the next stage to success.
func TestStageRunContinuesToNextStageAfterRestart(t *testing.T) {
interrupts := seamRestartRecovery(t, nil)
f := stopMidStage(t, ownedLiveStage)
var resumed atomic.Value
var secondRan atomic.Int32
r, s := f.restart(t, func(ctx context.Context, s *pipeline.Store, def stage.Definition) (stage.Result, error) {
if def.Key == "one" {
resumed.Store(stage.TakeResumeSession(ctx))
// The real client releases ownership once the session settled.
if err := s.RunSession(context.Background(), f.runID, 1, "ses_live", false, "v2", "ref-live"); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "finished while the server was down"}}, nil
}
secondRan.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next stage"}}, nil
})
if interrupts.Load() != 0 {
t.Fatal("start-up recovery interrupted a session that survived the restart")
}
r.reattachRuns()
r.wg.Wait()
d, err := s.GetRun(context.Background(), f.runID)
if err != nil {
t.Fatal(err)
}
if got, _ := resumed.Load().(string); got != "ses_live" {
t.Fatalf("stage one was not re-adopted from its session: %q", got)
}
if d.Status != "succeeded" || len(d.Steps) != 2 || secondRan.Load() != 1 {
t.Fatalf("run did not continue to the next stage: %+v", d)
}
one := d.Steps[0]
if one.Sequence != 1 || one.Status != "completed" || one.Output["markdown"] != "finished while the server was down" {
t.Fatalf("adopted visit not folded in: %+v", one)
}
if one.FileChanges == nil || one.FileChanges.Status != "partial" || !strings.Contains(one.FileChanges.Diagnostic, "restart") {
t.Fatalf("adopted visit claims a complete change capture: %+v", one.FileChanges)
}
}
// TestStageRunWithHarnessGoneFallsBackWithReason: when the harness process
// is gone after the restart, the run is interrupted as before, naming why.
func TestStageRunWithHarnessGoneFallsBackWithReason(t *testing.T) {
interrupts := seamRestartRecovery(t, fmt.Errorf("%w at http://127.0.0.1:1", opencode.ErrServiceNotRunning))
f := stopMidStage(t, ownedLiveStage)
r, s := f.restart(t, func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error) {
t.Error("a run whose harness was gone executed again")
return stage.Result{}, nil
})
r.reattachRuns()
r.wg.Wait()
if interrupts.Load() != 0 {
t.Fatal("recovery tried to interrupt a session on a service that is gone")
}
d, err := s.GetRun(context.Background(), f.runID)
if err != nil {
t.Fatal(err)
}
if d.Status != "interrupted" || !strings.Contains(d.Diagnostic, "was gone") || !strings.Contains(d.Steps[0].Diagnostic, "was gone") {
t.Fatalf("harness-gone fallback lacks a clear reason: status=%s run=%q step=%q", d.Status, d.Diagnostic, d.Steps[0].Diagnostic)
}
if owners, _ := s.ActiveSessionOwners(context.Background()); len(owners) != 0 {
t.Fatalf("a gone session is still recorded as active: %+v", owners)
}
}
// TestStageNotYetAdmittedStartsAgainAfterRestart: a visit the stopped server
// had begun but not handed to any session is started again, nothing replayed.
func TestStageNotYetAdmittedStartsAgainAfterRestart(t *testing.T) {
seamRestartRecovery(t, nil)
f := stopMidStage(t, func(ctx context.Context, _ *pipeline.Store, _ int64) (stage.Result, error) {
<-ctx.Done()
return stage.Result{}, ctx.Err()
})
var resumed atomic.Value
resumed.Store("")
r, s := f.restart(t, func(ctx context.Context, _ *pipeline.Store, def stage.Definition) (stage.Result, error) {
if def.Key == "one" {
resumed.Store(stage.TakeResumeSession(ctx))
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": def.Key}}, nil
})
r.reattachRuns()
r.wg.Wait()
d, _ := s.GetRun(context.Background(), f.runID)
if d.Status != "succeeded" || len(d.Steps) != 3 || d.Steps[0].Status != "interrupted" || d.Steps[1].Stage != "one" || d.Steps[1].Status != "completed" {
t.Fatalf("unadmitted stage was not started again: %+v", d)
}
if got := resumed.Load().(string); got != "" {
t.Fatalf("a fresh visit was treated as a re-adoption of %q", got)
}
}
// TestOperatorCancelStillStopsDetachingRun: only shutdown detaches; an
// operator cancel on a detaching server still settles the run as canceled.
func TestOperatorCancelStillStopsDetachingRun(t *testing.T) {
s, err := pipeline.Open(filepath.Join(t.TempDir(), "cancel.db"))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
if _, err := s.Submit(ctx, "cancel", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
r, err := newRunnerReattaching(ctx, s, t.TempDir(), true)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() }()
entered := make(chan struct{})
detached := &atomic.Bool{}
r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
close(entered)
<-ctx.Done()
detached.Store(stage.Detached(ctx))
return stage.Result{}, ctx.Err()
}
d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "cancel", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
<-entered
if err := r.cancel(d.ID); err != nil {
t.Fatal(err)
}
r.wg.Wait()
final, _ := s.GetRun(context.Background(), d.ID)
if detached.Load() || final.Status != "canceled" {
t.Fatalf("operator cancel did not stop the run: detached=%v %+v", detached.Load(), final)
}
}