aa8eddd686
A stopping server no longer interrupts harness sessions. Runs are canceled with a shutdown cause, so stage clients leave admitted sessions running and owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait, read back the result, continue the graph), restarts unadmitted visits, continues runs between stages, and falls back to interrupted with a reason when the harness is gone. Chats with live work are recorded on shutdown and reattached on start; turns that ended during the downtime are folded in from history. The server-spawned V2 chat serve is now a detached owned service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
263 lines
9.8 KiB
Go
263 lines
9.8 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/opencode"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// restartFixture runs a two-stage pipeline on a detaching runner, stops that
|
|
// "server" while stage one is in flight, and reopens the store for a second
|
|
// runner, the way a restart does.
|
|
type restartFixture struct {
|
|
path, root, workspace string
|
|
runID int64
|
|
}
|
|
|
|
// stopMidStage starts the run and shuts the first runner down while stage one
|
|
// is executing. firstStage is the first process's executor for stage one; it
|
|
// must block until its context ends.
|
|
func stopMidStage(t *testing.T, firstStage func(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error)) restartFixture {
|
|
t.Helper()
|
|
f := restartFixture{path: filepath.Join(t.TempDir(), "restart.db"), root: t.TempDir(), workspace: t.TempDir()}
|
|
s, err := pipeline.Open(f.path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, shutdown := context.WithCancel(context.Background())
|
|
if _, err := s.Submit(ctx, "restart", pipeline.Template([]string{"one", "two"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
r, err := newRunnerReattaching(ctx, s, f.root, true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.exportSession = validGateExport
|
|
entered := make(chan struct{})
|
|
var id atomic.Int64
|
|
r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
if def.Key != "one" {
|
|
t.Errorf("the first process reached stage %s", def.Key)
|
|
return stage.Result{}, fmt.Errorf("unexpected stage")
|
|
}
|
|
for id.Load() == 0 {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
close(entered)
|
|
return firstStage(ctx, s, id.Load())
|
|
}
|
|
d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "restart", Workspace: f.workspace, Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id.Store(d.ID)
|
|
f.runID = d.ID
|
|
<-entered
|
|
shutdown() // SIGTERM
|
|
r.wg.Wait()
|
|
r.schedulerWG.Wait()
|
|
persisted, err := s.GetRun(context.Background(), f.runID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if persisted.Status != protocol.RunStatusRunning || len(persisted.Steps) != 1 || persisted.Steps[0].Status != protocol.StepStatusRunning {
|
|
t.Fatalf("shutdown settled the run instead of leaving it in flight: %+v", persisted)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return f
|
|
}
|
|
|
|
// restart opens the second process's runner on the same database.
|
|
func (f restartFixture) restart(t *testing.T, execute func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error)) (*runner, *pipeline.Store) {
|
|
t.Helper()
|
|
s, err := pipeline.Open(f.path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
r, err := newRunnerReattaching(ctx, s, f.root, true)
|
|
if err != nil {
|
|
cancel()
|
|
s.Close()
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() })
|
|
r.exportSession = validGateExport
|
|
r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
return execute(ctx, s, def)
|
|
}
|
|
return r, s
|
|
}
|
|
|
|
func seamRestartRecovery(t *testing.T, alive error) *atomic.Int32 {
|
|
t.Helper()
|
|
interrupts := &atomic.Int32{}
|
|
previousAlive, previousInterrupt := stageServiceAlive, interruptRecoveredSession
|
|
stageServiceAlive = func(context.Context, pipeline.SessionOwner) error { return alive }
|
|
interruptRecoveredSession = func(context.Context, pipeline.SessionOwner) error { interrupts.Add(1); return nil }
|
|
t.Cleanup(func() { stageServiceAlive, interruptRecoveredSession = previousAlive, previousInterrupt })
|
|
return interrupts
|
|
}
|
|
|
|
// ownedLiveStage is a first-process stage one that admitted session
|
|
// ses_live (recorded like a real client does) and is still running at shutdown.
|
|
func ownedLiveStage(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error) {
|
|
if err := s.RunSession(context.Background(), id, 1, "ses_live", true, "v2", "ref-live"); err != nil {
|
|
return stage.Result{}, err
|
|
}
|
|
<-ctx.Done()
|
|
if stage.Detached(ctx) {
|
|
return stage.Result{}, stage.ErrDetached
|
|
}
|
|
return stage.Result{}, ctx.Err()
|
|
}
|
|
|
|
// TestStageRunContinuesToNextStageAfterRestart: the stage in flight at a server
|
|
// restart is re-adopted (never re-prompted, never interrupted), its result is
|
|
// folded in, and the run continues with the next stage to success.
|
|
func TestStageRunContinuesToNextStageAfterRestart(t *testing.T) {
|
|
interrupts := seamRestartRecovery(t, nil)
|
|
f := stopMidStage(t, ownedLiveStage)
|
|
var resumed atomic.Value
|
|
var secondRan atomic.Int32
|
|
r, s := f.restart(t, func(ctx context.Context, s *pipeline.Store, def stage.Definition) (stage.Result, error) {
|
|
if def.Key == "one" {
|
|
resumed.Store(stage.TakeResumeSession(ctx))
|
|
// The real client releases ownership once the session settled.
|
|
if err := s.RunSession(context.Background(), f.runID, 1, "ses_live", false, "v2", "ref-live"); err != nil {
|
|
return stage.Result{}, err
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "finished while the server was down"}}, nil
|
|
}
|
|
secondRan.Add(1)
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next stage"}}, nil
|
|
})
|
|
if interrupts.Load() != 0 {
|
|
t.Fatal("start-up recovery interrupted a session that survived the restart")
|
|
}
|
|
r.reattachRuns()
|
|
r.wg.Wait()
|
|
d, err := s.GetRun(context.Background(), f.runID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, _ := resumed.Load().(string); got != "ses_live" {
|
|
t.Fatalf("stage one was not re-adopted from its session: %q", got)
|
|
}
|
|
if d.Status != "succeeded" || len(d.Steps) != 2 || secondRan.Load() != 1 {
|
|
t.Fatalf("run did not continue to the next stage: %+v", d)
|
|
}
|
|
one := d.Steps[0]
|
|
if one.Sequence != 1 || one.Status != "completed" || one.Output["markdown"] != "finished while the server was down" {
|
|
t.Fatalf("adopted visit not folded in: %+v", one)
|
|
}
|
|
if one.FileChanges == nil || one.FileChanges.Status != "partial" || !strings.Contains(one.FileChanges.Diagnostic, "restart") {
|
|
t.Fatalf("adopted visit claims a complete change capture: %+v", one.FileChanges)
|
|
}
|
|
}
|
|
|
|
// TestStageRunWithHarnessGoneFallsBackWithReason: when the harness process
|
|
// is gone after the restart, the run is interrupted as before, naming why.
|
|
func TestStageRunWithHarnessGoneFallsBackWithReason(t *testing.T) {
|
|
interrupts := seamRestartRecovery(t, fmt.Errorf("%w at http://127.0.0.1:1", opencode.ErrServiceNotRunning))
|
|
f := stopMidStage(t, ownedLiveStage)
|
|
r, s := f.restart(t, func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error) {
|
|
t.Error("a run whose harness was gone executed again")
|
|
return stage.Result{}, nil
|
|
})
|
|
r.reattachRuns()
|
|
r.wg.Wait()
|
|
if interrupts.Load() != 0 {
|
|
t.Fatal("recovery tried to interrupt a session on a service that is gone")
|
|
}
|
|
d, err := s.GetRun(context.Background(), f.runID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if d.Status != "interrupted" || !strings.Contains(d.Diagnostic, "was gone") || !strings.Contains(d.Steps[0].Diagnostic, "was gone") {
|
|
t.Fatalf("harness-gone fallback lacks a clear reason: status=%s run=%q step=%q", d.Status, d.Diagnostic, d.Steps[0].Diagnostic)
|
|
}
|
|
if owners, _ := s.ActiveSessionOwners(context.Background()); len(owners) != 0 {
|
|
t.Fatalf("a gone session is still recorded as active: %+v", owners)
|
|
}
|
|
}
|
|
|
|
// TestStageNotYetAdmittedStartsAgainAfterRestart: a visit the stopped server
|
|
// had begun but not handed to any session is started again, nothing replayed.
|
|
func TestStageNotYetAdmittedStartsAgainAfterRestart(t *testing.T) {
|
|
seamRestartRecovery(t, nil)
|
|
f := stopMidStage(t, func(ctx context.Context, _ *pipeline.Store, _ int64) (stage.Result, error) {
|
|
<-ctx.Done()
|
|
return stage.Result{}, ctx.Err()
|
|
})
|
|
var resumed atomic.Value
|
|
resumed.Store("")
|
|
r, s := f.restart(t, func(ctx context.Context, _ *pipeline.Store, def stage.Definition) (stage.Result, error) {
|
|
if def.Key == "one" {
|
|
resumed.Store(stage.TakeResumeSession(ctx))
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": def.Key}}, nil
|
|
})
|
|
r.reattachRuns()
|
|
r.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), f.runID)
|
|
if d.Status != "succeeded" || len(d.Steps) != 3 || d.Steps[0].Status != "interrupted" || d.Steps[1].Stage != "one" || d.Steps[1].Status != "completed" {
|
|
t.Fatalf("unadmitted stage was not started again: %+v", d)
|
|
}
|
|
if got := resumed.Load().(string); got != "" {
|
|
t.Fatalf("a fresh visit was treated as a re-adoption of %q", got)
|
|
}
|
|
}
|
|
|
|
// TestOperatorCancelStillStopsDetachingRun: only shutdown detaches; an
|
|
// operator cancel on a detaching server still settles the run as canceled.
|
|
func TestOperatorCancelStillStopsDetachingRun(t *testing.T) {
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "cancel.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
if _, err := s.Submit(ctx, "cancel", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
r, err := newRunnerReattaching(ctx, s, t.TempDir(), true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() }()
|
|
entered := make(chan struct{})
|
|
detached := &atomic.Bool{}
|
|
r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
close(entered)
|
|
<-ctx.Done()
|
|
detached.Store(stage.Detached(ctx))
|
|
return stage.Result{}, ctx.Err()
|
|
}
|
|
d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "cancel", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
<-entered
|
|
if err := r.cancel(d.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
final, _ := s.GetRun(context.Background(), d.ID)
|
|
if detached.Load() || final.Status != "canceled" {
|
|
t.Fatalf("operator cancel did not stop the run: detached=%v %+v", detached.Load(), final)
|
|
}
|
|
}
|