package server import ( "context" "fmt" "path/filepath" "strings" "sync/atomic" "testing" "time" "git.a-wan.net/alan/pipelines/internal/opencode" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) // restartFixture runs a two-stage pipeline on a detaching runner, stops that // "server" while stage one is in flight, and reopens the store for a second // runner, the way a restart does. type restartFixture struct { path, root, workspace string runID int64 } // stopMidStage starts the run and shuts the first runner down while stage one // is executing. firstStage is the first process's executor for stage one; it // must block until its context ends. func stopMidStage(t *testing.T, firstStage func(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error)) restartFixture { t.Helper() f := restartFixture{path: filepath.Join(t.TempDir(), "restart.db"), root: t.TempDir(), workspace: t.TempDir()} s, err := pipeline.Open(f.path) if err != nil { t.Fatal(err) } ctx, shutdown := context.WithCancel(context.Background()) if _, err := s.Submit(ctx, "restart", pipeline.Template([]string{"one", "two"}, true)); err != nil { t.Fatal(err) } items, _ := s.List(ctx) r, err := newRunnerReattaching(ctx, s, f.root, true) if err != nil { t.Fatal(err) } r.exportSession = validGateExport entered := make(chan struct{}) var id atomic.Int64 r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { if def.Key != "one" { t.Errorf("the first process reached stage %s", def.Key) return stage.Result{}, fmt.Errorf("unexpected stage") } for id.Load() == 0 { time.Sleep(time.Millisecond) } close(entered) return firstStage(ctx, s, id.Load()) } d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "restart", Workspace: f.workspace, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } id.Store(d.ID) f.runID = d.ID <-entered shutdown() // SIGTERM r.wg.Wait() r.schedulerWG.Wait() persisted, err := s.GetRun(context.Background(), f.runID) if err != nil { t.Fatal(err) } if persisted.Status != protocol.RunStatusRunning || len(persisted.Steps) != 1 || persisted.Steps[0].Status != protocol.StepStatusRunning { t.Fatalf("shutdown settled the run instead of leaving it in flight: %+v", persisted) } if err := s.Close(); err != nil { t.Fatal(err) } return f } // restart opens the second process's runner on the same database. func (f restartFixture) restart(t *testing.T, execute func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error)) (*runner, *pipeline.Store) { t.Helper() s, err := pipeline.Open(f.path) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) r, err := newRunnerReattaching(ctx, s, f.root, true) if err != nil { cancel() s.Close() t.Fatal(err) } t.Cleanup(func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() }) r.exportSession = validGateExport r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { return execute(ctx, s, def) } return r, s } func seamRestartRecovery(t *testing.T, alive error) *atomic.Int32 { t.Helper() interrupts := &atomic.Int32{} previousAlive, previousInterrupt := stageServiceAlive, interruptRecoveredSession stageServiceAlive = func(context.Context, pipeline.SessionOwner) error { return alive } interruptRecoveredSession = func(context.Context, pipeline.SessionOwner) error { interrupts.Add(1); return nil } t.Cleanup(func() { stageServiceAlive, interruptRecoveredSession = previousAlive, previousInterrupt }) return interrupts } // ownedLiveStage is a first-process stage one that admitted session // ses_live (recorded like a real client does) and is still running at shutdown. func ownedLiveStage(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error) { if err := s.RunSession(context.Background(), id, 1, "ses_live", true, "v2", "ref-live"); err != nil { return stage.Result{}, err } <-ctx.Done() if stage.Detached(ctx) { return stage.Result{}, stage.ErrDetached } return stage.Result{}, ctx.Err() } // TestStageRunContinuesToNextStageAfterRestart: the stage in flight at a server // restart is re-adopted (never re-prompted, never interrupted), its result is // folded in, and the run continues with the next stage to success. func TestStageRunContinuesToNextStageAfterRestart(t *testing.T) { interrupts := seamRestartRecovery(t, nil) f := stopMidStage(t, ownedLiveStage) var resumed atomic.Value var secondRan atomic.Int32 r, s := f.restart(t, func(ctx context.Context, s *pipeline.Store, def stage.Definition) (stage.Result, error) { if def.Key == "one" { resumed.Store(stage.TakeResumeSession(ctx)) // The real client releases ownership once the session settled. if err := s.RunSession(context.Background(), f.runID, 1, "ses_live", false, "v2", "ref-live"); err != nil { return stage.Result{}, err } return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "finished while the server was down"}}, nil } secondRan.Add(1) return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next stage"}}, nil }) if interrupts.Load() != 0 { t.Fatal("start-up recovery interrupted a session that survived the restart") } r.reattachRuns() r.wg.Wait() d, err := s.GetRun(context.Background(), f.runID) if err != nil { t.Fatal(err) } if got, _ := resumed.Load().(string); got != "ses_live" { t.Fatalf("stage one was not re-adopted from its session: %q", got) } if d.Status != "succeeded" || len(d.Steps) != 2 || secondRan.Load() != 1 { t.Fatalf("run did not continue to the next stage: %+v", d) } one := d.Steps[0] if one.Sequence != 1 || one.Status != "completed" || one.Output["markdown"] != "finished while the server was down" { t.Fatalf("adopted visit not folded in: %+v", one) } if one.FileChanges == nil || one.FileChanges.Status != "partial" || !strings.Contains(one.FileChanges.Diagnostic, "restart") { t.Fatalf("adopted visit claims a complete change capture: %+v", one.FileChanges) } } // TestStageRunWithHarnessGoneFallsBackWithReason: when the harness process // is gone after the restart, the run is interrupted as before, naming why. func TestStageRunWithHarnessGoneFallsBackWithReason(t *testing.T) { interrupts := seamRestartRecovery(t, fmt.Errorf("%w at http://127.0.0.1:1", opencode.ErrServiceNotRunning)) f := stopMidStage(t, ownedLiveStage) r, s := f.restart(t, func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error) { t.Error("a run whose harness was gone executed again") return stage.Result{}, nil }) r.reattachRuns() r.wg.Wait() if interrupts.Load() != 0 { t.Fatal("recovery tried to interrupt a session on a service that is gone") } d, err := s.GetRun(context.Background(), f.runID) if err != nil { t.Fatal(err) } if d.Status != "interrupted" || !strings.Contains(d.Diagnostic, "was gone") || !strings.Contains(d.Steps[0].Diagnostic, "was gone") { t.Fatalf("harness-gone fallback lacks a clear reason: status=%s run=%q step=%q", d.Status, d.Diagnostic, d.Steps[0].Diagnostic) } if owners, _ := s.ActiveSessionOwners(context.Background()); len(owners) != 0 { t.Fatalf("a gone session is still recorded as active: %+v", owners) } } // TestStageNotYetAdmittedStartsAgainAfterRestart: a visit the stopped server // had begun but not handed to any session is started again, nothing replayed. func TestStageNotYetAdmittedStartsAgainAfterRestart(t *testing.T) { seamRestartRecovery(t, nil) f := stopMidStage(t, func(ctx context.Context, _ *pipeline.Store, _ int64) (stage.Result, error) { <-ctx.Done() return stage.Result{}, ctx.Err() }) var resumed atomic.Value resumed.Store("") r, s := f.restart(t, func(ctx context.Context, _ *pipeline.Store, def stage.Definition) (stage.Result, error) { if def.Key == "one" { resumed.Store(stage.TakeResumeSession(ctx)) } return stage.Result{Outcome: "success", Output: stage.Values{"markdown": def.Key}}, nil }) r.reattachRuns() r.wg.Wait() d, _ := s.GetRun(context.Background(), f.runID) if d.Status != "succeeded" || len(d.Steps) != 3 || d.Steps[0].Status != "interrupted" || d.Steps[1].Stage != "one" || d.Steps[1].Status != "completed" { t.Fatalf("unadmitted stage was not started again: %+v", d) } if got := resumed.Load().(string); got != "" { t.Fatalf("a fresh visit was treated as a re-adoption of %q", got) } } // TestOperatorCancelStillStopsDetachingRun: only shutdown detaches; an // operator cancel on a detaching server still settles the run as canceled. func TestOperatorCancelStillStopsDetachingRun(t *testing.T) { s, err := pipeline.Open(filepath.Join(t.TempDir(), "cancel.db")) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) if _, err := s.Submit(ctx, "cancel", pipeline.Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := s.List(ctx) r, err := newRunnerReattaching(ctx, s, t.TempDir(), true) if err != nil { t.Fatal(err) } defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() }() entered := make(chan struct{}) detached := &atomic.Bool{} r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { close(entered) <-ctx.Done() detached.Store(stage.Detached(ctx)) return stage.Result{}, ctx.Err() } d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "cancel", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } <-entered if err := r.cancel(d.ID); err != nil { t.Fatal(err) } r.wg.Wait() final, _ := s.GetRun(context.Background(), d.ID) if detached.Load() || final.Status != "canceled" { t.Fatalf("operator cancel did not stop the run: detached=%v %+v", detached.Load(), final) } }