From 3bfb3d6b44489de0fb1708cada8e8d6755eb2ba5 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 1 Nov 2026 18:58:14 +0000 Subject: [PATCH] Fix decide-after-finish refusal and run size loop data race A run stays in r.active after finishRunState persists its terminal status until its goroutine finishes notifying, so a decision in that window was refused as "already active". decide now refuses only runs whose terminal state is not yet durable; a finished run gets ResolveDecision's already-decided / not-waiting answer, and an executing run is still refused. Tests overwrote runner.artifactsRoot after Handler/newRunner had started background goroutines (sizeLoop, session export jobs) that read it. The root is now fixed at construction through newRunnerAt, which HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it there instead of mutating the field afterwards. --- internal/server/artifacts_test.go | 3 +- internal/server/deliverable_test.go | 3 +- internal/server/human_test.go | 60 +++++++++++++++++++ internal/server/runner.go | 22 ++++++- .../server/session_export_exhausted_test.go | 3 +- internal/server/session_export_gate_test.go | 6 +- internal/server/session_export_jobs_test.go | 3 +- internal/server/session_export_live_test.go | 3 +- internal/server/session_export_test.go | 3 +- internal/server/websocket.go | 5 +- 10 files changed, 89 insertions(+), 22 deletions(-) diff --git a/internal/server/artifacts_test.go b/internal/server/artifacts_test.go index da5e8bb9..5d69ca06 100644 --- a/internal/server/artifacts_test.go +++ b/internal/server/artifacts_test.go @@ -215,9 +215,8 @@ func runWithArtifact(t *testing.T) (*runHandler, protocol.RunStep) { if err != nil { t.Fatal(err) } - handler := Handler(ctx, store).(*runHandler) + handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, store).(*runHandler) t.Cleanup(func() { cancel(); WaitRuns(handler) }) - handler.runner.artifactsRoot = t.TempDir() handler.runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { if def.Turn.Private == "" || def.Turn.Public == "" { t.Error("runner did not provide the turn artifact scopes") diff --git a/internal/server/deliverable_test.go b/internal/server/deliverable_test.go index ceb026ca..8a055874 100644 --- a/internal/server/deliverable_test.go +++ b/internal/server/deliverable_test.go @@ -154,12 +154,11 @@ func TestOutputFormatRetryInheritanceThroughRuntime(t *testing.T) { t.Fatal(err) } items, _ := s.List(ctx) - r, err := newRunner(ctx, s) + r, err := newRunnerAt(ctx, s, t.TempDir()) if err != nil { t.Fatal(err) } defer func() { cancel(); r.wg.Wait() }() - r.artifactsRoot = t.TempDir() r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{}, fmt.Errorf("fixture failure") } diff --git a/internal/server/human_test.go b/internal/server/human_test.go index 30d9307c..c4c2a5ea 100644 --- a/internal/server/human_test.go +++ b/internal/server/human_test.go @@ -431,3 +431,63 @@ func TestHumanStageWaitingNotification(t *testing.T) { t.Fatalf("waiting notification receipt missing: %+v", detail.Steps[0].Notifications) } } + +// TestDecideOnFinishingRunIsAnsweredNotRefused pins the window between +// finishRunState persisting a terminal status and the launch teardown removing +// the run from r.active: a decision then gets the store's idempotent/not-waiting +// answer, while a run that is still executing is refused as active. +func TestDecideOnFinishingRunIsAnsweredNotRefused(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "human.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + r, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + defer r.wg.Wait() + r.execute = successExecute + run := startHumanRun(t, ctx, r, store, "human-finishing") + + // A waiting run that is genuinely executing (registered, not finished) is + // refused. + r.mu.Lock() + r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace} + r.mu.Unlock() + if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt"}); err == nil || !strings.Contains(err.Error(), "already active") { + t.Fatalf("decision on an executing run was not refused: %v", err) + } + r.mu.Lock() + delete(r.active, run.ID) + r.mu.Unlock() + + if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt", Feedback: "stop"}); err != nil { + t.Fatal(err) + } + r.wg.Wait() + if detail, err := store.GetRun(ctx, run.ID); err != nil || detail.Status != "succeeded" { + t.Fatalf("run did not finish: %+v %v", detail, err) + } + + // Simulate the finishing window: terminal status persisted, reservation not + // yet released. + r.mu.Lock() + r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace} + r.mu.Unlock() + repeat, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "halt"}) + if err != nil { + t.Fatalf("repeat decision on a finished run was refused: %v", err) + } + if repeat.Status != "succeeded" || repeat.Steps[0].Output["feedback"] != "stop" { + t.Fatalf("repeat decision changed the run: %+v", repeat) + } + if _, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "go"}); err == nil || !strings.Contains(err.Error(), "not waiting") { + t.Fatalf("conflicting decision on a finished run did not report not waiting: %v", err) + } + r.mu.Lock() + delete(r.active, run.ID) + r.mu.Unlock() +} diff --git a/internal/server/runner.go b/internal/server/runner.go index 2b9c13eb..6c4dd2bb 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -87,6 +87,15 @@ type runner struct { } func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { + return newRunnerAt(ctx, s, "") +} + +// newRunnerAt builds a runner whose stage artifacts live under artifactsRoot +// ("" means /artifacts). artifactsRoot is fixed before any +// background goroutine (session export jobs, notifications, the size backfill) +// starts and is never written afterwards, so those goroutines read it without +// locking. +func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (*runner, error) { if err := s.AcquireRunnerLock(); err != nil { return nil, err } @@ -123,7 +132,10 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { if err := s.RecoverRuns(ctx); err != nil { return nil, err } - root := filepath.Join(s.Directory(), "artifacts") + root := artifactsRoot + if root == "" { + root = filepath.Join(s.Directory(), "artifacts") + } active := map[int64]activeRun{} rows, err := s.ListRuns(ctx) if err != nil { @@ -780,7 +792,13 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) { r.mu.Unlock() return nil, err } - if _, busy := r.active[detail.ID]; busy { + // finishRunState persists the terminal status under r.mu, but the run stays + // in r.active until its goroutine finishes notifying and the launch teardown + // runs. A run whose terminal state is already durable (FinishedAt is only set + // once cleanup is confirmed) has no executor left to race with a decision, so + // ResolveDecision answers it ("already decided" or "not waiting") and never + // resumes a terminal run. A run that is still executing is refused. + if _, busy := r.active[detail.ID]; busy && detail.FinishedAt == "" { r.mu.Unlock() return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID) } diff --git a/internal/server/session_export_exhausted_test.go b/internal/server/session_export_exhausted_test.go index c401081c..6e06143a 100644 --- a/internal/server/session_export_exhausted_test.go +++ b/internal/server/session_export_exhausted_test.go @@ -112,12 +112,11 @@ func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) { t.Fatalf("fixture is not a parked export: %+v", parked) } ctx, cancel := context.WithCancel(context.Background()) - r, err := newRunner(ctx, reopened) + r, err := newRunnerAt(ctx, reopened, root) if err != nil { t.Fatal(err) } defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }() - r.artifactsRoot = root r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { if def.Key != "two" { t.Errorf("replayed owner: %s", def.Key) diff --git a/internal/server/session_export_gate_test.go b/internal/server/session_export_gate_test.go index 3c9519c4..2011cce4 100644 --- a/internal/server/session_export_gate_test.go +++ b/internal/server/session_export_gate_test.go @@ -27,9 +27,8 @@ func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRe if err != nil { t.Fatal(err) } - h := Handler(ctx, s).(*runHandler) + h := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, s).(*runHandler) t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() }) - h.runner.artifactsRoot = t.TempDir() h.runner.exportSession = export if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil { t.Fatal(err) @@ -352,12 +351,11 @@ func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) - r, err := newRunner(ctx, reopened) + r, err := newRunnerAt(ctx, reopened, root) if err != nil { t.Fatal(err) } defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }() - r.artifactsRoot = root r.exportSession = validGateExport r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { if def.Key != "two" { diff --git a/internal/server/session_export_jobs_test.go b/internal/server/session_export_jobs_test.go index 2669adaa..bd4e06a7 100644 --- a/internal/server/session_export_jobs_test.go +++ b/internal/server/session_export_jobs_test.go @@ -183,12 +183,11 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t ctx, cancel := context.WithCancel(context.Background()) defer cancel() sessionExportCLI(t) // read-only native CLI fixture, never a live model/run - r, err := newRunner(ctx, s) + r, err := newRunnerAt(ctx, s, root) if err != nil { t.Fatal(err) } defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }() - r.artifactsRoot = root r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { t.Error("restart recovery reran a stage") return stage.Result{}, errors.New("forbidden rerun") diff --git a/internal/server/session_export_live_test.go b/internal/server/session_export_live_test.go index b05c531a..a27c8ee6 100644 --- a/internal/server/session_export_live_test.go +++ b/internal/server/session_export_live_test.go @@ -66,8 +66,7 @@ func TestSessionExportLive(t *testing.T) { if err != nil { t.Fatal(err) } - handler := Handler(ctx, store).(*runHandler) - handler.runner.artifactsRoot = filepath.Join(root, "artifacts") + handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: filepath.Join(root, "artifacts")}, store).(*runHandler) defer func() { cancel(); WaitRuns(handler) }() workspace := filepath.Join(root, "ws") if err := os.Mkdir(workspace, 0700); err != nil { diff --git a/internal/server/session_export_test.go b/internal/server/session_export_test.go index 6ec7fcea..5044ea7f 100644 --- a/internal/server/session_export_test.go +++ b/internal/server/session_export_test.go @@ -95,9 +95,8 @@ func sessionExportRunner(t *testing.T, backend string, seam func(stage.SessionEx if err != nil { t.Fatal(err) } - handler := Handler(ctx, store).(*runHandler) + handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, store).(*runHandler) t.Cleanup(func() { cancel(); WaitRuns(handler) }) - handler.runner.artifactsRoot = t.TempDir() if seam != nil { handler.runner.exportSession = func(_ context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) { return seam(request) diff --git a/internal/server/websocket.go b/internal/server/websocket.go index eecffe18..db5765b9 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -77,13 +77,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin var runs *runner var runErr error if len(stores) > 0 { - runs, runErr = newRunner(ctx, stores[0]) + runs, runErr = newRunnerAt(ctx, stores[0], options.ArtifactsRoot) if runErr == nil { runs.events = events } - if runErr == nil && options.ArtifactsRoot != "" { - runs.artifactsRoot = options.ArtifactsRoot - } if runErr == nil && options.RunTitles != nil { runs.title = options.RunTitles }