diff --git a/internal/server/artifacts_test.go b/internal/server/artifacts_test.go index da5e8bb9..5d69ca06 100644 --- a/internal/server/artifacts_test.go +++ b/internal/server/artifacts_test.go @@ -215,9 +215,8 @@ func runWithArtifact(t *testing.T) (*runHandler, protocol.RunStep) { if err != nil { t.Fatal(err) } - handler := Handler(ctx, store).(*runHandler) + handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, store).(*runHandler) t.Cleanup(func() { cancel(); WaitRuns(handler) }) - handler.runner.artifactsRoot = t.TempDir() handler.runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { if def.Turn.Private == "" || def.Turn.Public == "" { t.Error("runner did not provide the turn artifact scopes") diff --git a/internal/server/deliverable_test.go b/internal/server/deliverable_test.go index ceb026ca..8a055874 100644 --- a/internal/server/deliverable_test.go +++ b/internal/server/deliverable_test.go @@ -154,12 +154,11 @@ func TestOutputFormatRetryInheritanceThroughRuntime(t *testing.T) { t.Fatal(err) } items, _ := s.List(ctx) - r, err := newRunner(ctx, s) + r, err := newRunnerAt(ctx, s, t.TempDir()) if err != nil { t.Fatal(err) } defer func() { cancel(); r.wg.Wait() }() - r.artifactsRoot = t.TempDir() r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{}, fmt.Errorf("fixture failure") } diff --git a/internal/server/human_test.go b/internal/server/human_test.go index 30d9307c..c4c2a5ea 100644 --- a/internal/server/human_test.go +++ b/internal/server/human_test.go @@ -431,3 +431,63 @@ func TestHumanStageWaitingNotification(t *testing.T) { t.Fatalf("waiting notification receipt missing: %+v", detail.Steps[0].Notifications) } } + +// TestDecideOnFinishingRunIsAnsweredNotRefused pins the window between +// finishRunState persisting a terminal status and the launch teardown removing +// the run from r.active: a decision then gets the store's idempotent/not-waiting +// answer, while a run that is still executing is refused as active. +func TestDecideOnFinishingRunIsAnsweredNotRefused(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "human.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + r, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + defer r.wg.Wait() + r.execute = successExecute + run := startHumanRun(t, ctx, r, store, "human-finishing") + + // A waiting run that is genuinely executing (registered, not finished) is + // refused. + r.mu.Lock() + r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace} + r.mu.Unlock() + if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt"}); err == nil || !strings.Contains(err.Error(), "already active") { + t.Fatalf("decision on an executing run was not refused: %v", err) + } + r.mu.Lock() + delete(r.active, run.ID) + r.mu.Unlock() + + if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt", Feedback: "stop"}); err != nil { + t.Fatal(err) + } + r.wg.Wait() + if detail, err := store.GetRun(ctx, run.ID); err != nil || detail.Status != "succeeded" { + t.Fatalf("run did not finish: %+v %v", detail, err) + } + + // Simulate the finishing window: terminal status persisted, reservation not + // yet released. + r.mu.Lock() + r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace} + r.mu.Unlock() + repeat, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "halt"}) + if err != nil { + t.Fatalf("repeat decision on a finished run was refused: %v", err) + } + if repeat.Status != "succeeded" || repeat.Steps[0].Output["feedback"] != "stop" { + t.Fatalf("repeat decision changed the run: %+v", repeat) + } + if _, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "go"}); err == nil || !strings.Contains(err.Error(), "not waiting") { + t.Fatalf("conflicting decision on a finished run did not report not waiting: %v", err) + } + r.mu.Lock() + delete(r.active, run.ID) + r.mu.Unlock() +} diff --git a/internal/server/runner.go b/internal/server/runner.go index 2b9c13eb..6c4dd2bb 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -87,6 +87,15 @@ type runner struct { } func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { + return newRunnerAt(ctx, s, "") +} + +// newRunnerAt builds a runner whose stage artifacts live under artifactsRoot +// ("" means /artifacts). artifactsRoot is fixed before any +// background goroutine (session export jobs, notifications, the size backfill) +// starts and is never written afterwards, so those goroutines read it without +// locking. +func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (*runner, error) { if err := s.AcquireRunnerLock(); err != nil { return nil, err } @@ -123,7 +132,10 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { if err := s.RecoverRuns(ctx); err != nil { return nil, err } - root := filepath.Join(s.Directory(), "artifacts") + root := artifactsRoot + if root == "" { + root = filepath.Join(s.Directory(), "artifacts") + } active := map[int64]activeRun{} rows, err := s.ListRuns(ctx) if err != nil { @@ -780,7 +792,13 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) { r.mu.Unlock() return nil, err } - if _, busy := r.active[detail.ID]; busy { + // finishRunState persists the terminal status under r.mu, but the run stays + // in r.active until its goroutine finishes notifying and the launch teardown + // runs. A run whose terminal state is already durable (FinishedAt is only set + // once cleanup is confirmed) has no executor left to race with a decision, so + // ResolveDecision answers it ("already decided" or "not waiting") and never + // resumes a terminal run. A run that is still executing is refused. + if _, busy := r.active[detail.ID]; busy && detail.FinishedAt == "" { r.mu.Unlock() return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID) } diff --git a/internal/server/session_export_exhausted_test.go b/internal/server/session_export_exhausted_test.go index c401081c..6e06143a 100644 --- a/internal/server/session_export_exhausted_test.go +++ b/internal/server/session_export_exhausted_test.go @@ -112,12 +112,11 @@ func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) { t.Fatalf("fixture is not a parked export: %+v", parked) } ctx, cancel := context.WithCancel(context.Background()) - r, err := newRunner(ctx, reopened) + r, err := newRunnerAt(ctx, reopened, root) if err != nil { t.Fatal(err) } defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }() - r.artifactsRoot = root r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { if def.Key != "two" { t.Errorf("replayed owner: %s", def.Key) diff --git a/internal/server/session_export_gate_test.go b/internal/server/session_export_gate_test.go index 3c9519c4..2011cce4 100644 --- a/internal/server/session_export_gate_test.go +++ b/internal/server/session_export_gate_test.go @@ -27,9 +27,8 @@ func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRe if err != nil { t.Fatal(err) } - h := Handler(ctx, s).(*runHandler) + h := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, s).(*runHandler) t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() }) - h.runner.artifactsRoot = t.TempDir() h.runner.exportSession = export if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil { t.Fatal(err) @@ -352,12 +351,11 @@ func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) - r, err := newRunner(ctx, reopened) + r, err := newRunnerAt(ctx, reopened, root) if err != nil { t.Fatal(err) } defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }() - r.artifactsRoot = root r.exportSession = validGateExport r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { if def.Key != "two" { diff --git a/internal/server/session_export_jobs_test.go b/internal/server/session_export_jobs_test.go index 2669adaa..bd4e06a7 100644 --- a/internal/server/session_export_jobs_test.go +++ b/internal/server/session_export_jobs_test.go @@ -183,12 +183,11 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t ctx, cancel := context.WithCancel(context.Background()) defer cancel() sessionExportCLI(t) // read-only native CLI fixture, never a live model/run - r, err := newRunner(ctx, s) + r, err := newRunnerAt(ctx, s, root) if err != nil { t.Fatal(err) } defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }() - r.artifactsRoot = root r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { t.Error("restart recovery reran a stage") return stage.Result{}, errors.New("forbidden rerun") diff --git a/internal/server/session_export_live_test.go b/internal/server/session_export_live_test.go index b05c531a..a27c8ee6 100644 --- a/internal/server/session_export_live_test.go +++ b/internal/server/session_export_live_test.go @@ -66,8 +66,7 @@ func TestSessionExportLive(t *testing.T) { if err != nil { t.Fatal(err) } - handler := Handler(ctx, store).(*runHandler) - handler.runner.artifactsRoot = filepath.Join(root, "artifacts") + handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: filepath.Join(root, "artifacts")}, store).(*runHandler) defer func() { cancel(); WaitRuns(handler) }() workspace := filepath.Join(root, "ws") if err := os.Mkdir(workspace, 0700); err != nil { diff --git a/internal/server/session_export_test.go b/internal/server/session_export_test.go index 6ec7fcea..5044ea7f 100644 --- a/internal/server/session_export_test.go +++ b/internal/server/session_export_test.go @@ -95,9 +95,8 @@ func sessionExportRunner(t *testing.T, backend string, seam func(stage.SessionEx if err != nil { t.Fatal(err) } - handler := Handler(ctx, store).(*runHandler) + handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, store).(*runHandler) t.Cleanup(func() { cancel(); WaitRuns(handler) }) - handler.runner.artifactsRoot = t.TempDir() if seam != nil { handler.runner.exportSession = func(_ context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) { return seam(request) diff --git a/internal/server/websocket.go b/internal/server/websocket.go index eecffe18..db5765b9 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -77,13 +77,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin var runs *runner var runErr error if len(stores) > 0 { - runs, runErr = newRunner(ctx, stores[0]) + runs, runErr = newRunnerAt(ctx, stores[0], options.ArtifactsRoot) if runErr == nil { runs.events = events } - if runErr == nil && options.ArtifactsRoot != "" { - runs.artifactsRoot = options.ArtifactsRoot - } if runErr == nil && options.RunTitles != nil { runs.title = options.RunTitles }