Fix decide-after-finish refusal and run size loop data race

A run stays in r.active after finishRunState persists its terminal status
until its goroutine finishes notifying, so a decision in that window was
refused as "already active". decide now refuses only runs whose terminal
state is not yet durable; a finished run gets ResolveDecision's
already-decided / not-waiting answer, and an executing run is still refused.

Tests overwrote runner.artifactsRoot after Handler/newRunner had started
background goroutines (sizeLoop, session export jobs) that read it. The
root is now fixed at construction through newRunnerAt, which
HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it
there instead of mutating the field afterwards.
This commit is contained in:
2026-11-01 18:58:14 +00:00
parent c41f5caa4f
commit 3bfb3d6b44
10 changed files with 89 additions and 22 deletions
+1 -2
View File
@@ -215,9 +215,8 @@ func runWithArtifact(t *testing.T) (*runHandler, protocol.RunStep) {
if err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store).(*runHandler)
handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, store).(*runHandler)
t.Cleanup(func() { cancel(); WaitRuns(handler) })
handler.runner.artifactsRoot = t.TempDir()
handler.runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Turn.Private == "" || def.Turn.Public == "" {
t.Error("runner did not provide the turn artifact scopes")
+1 -2
View File
@@ -154,12 +154,11 @@ func TestOutputFormatRetryInheritanceThroughRuntime(t *testing.T) {
t.Fatal(err)
}
items, _ := s.List(ctx)
r, err := newRunner(ctx, s)
r, err := newRunnerAt(ctx, s, t.TempDir())
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait() }()
r.artifactsRoot = t.TempDir()
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, fmt.Errorf("fixture failure")
}
+60
View File
@@ -431,3 +431,63 @@ func TestHumanStageWaitingNotification(t *testing.T) {
t.Fatalf("waiting notification receipt missing: %+v", detail.Steps[0].Notifications)
}
}
// TestDecideOnFinishingRunIsAnsweredNotRefused pins the window between
// finishRunState persisting a terminal status and the launch teardown removing
// the run from r.active: a decision then gets the store's idempotent/not-waiting
// answer, while a run that is still executing is refused as active.
func TestDecideOnFinishingRunIsAnsweredNotRefused(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
r.execute = successExecute
run := startHumanRun(t, ctx, r, store, "human-finishing")
// A waiting run that is genuinely executing (registered, not finished) is
// refused.
r.mu.Lock()
r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace}
r.mu.Unlock()
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt"}); err == nil || !strings.Contains(err.Error(), "already active") {
t.Fatalf("decision on an executing run was not refused: %v", err)
}
r.mu.Lock()
delete(r.active, run.ID)
r.mu.Unlock()
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt", Feedback: "stop"}); err != nil {
t.Fatal(err)
}
r.wg.Wait()
if detail, err := store.GetRun(ctx, run.ID); err != nil || detail.Status != "succeeded" {
t.Fatalf("run did not finish: %+v %v", detail, err)
}
// Simulate the finishing window: terminal status persisted, reservation not
// yet released.
r.mu.Lock()
r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace}
r.mu.Unlock()
repeat, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "halt"})
if err != nil {
t.Fatalf("repeat decision on a finished run was refused: %v", err)
}
if repeat.Status != "succeeded" || repeat.Steps[0].Output["feedback"] != "stop" {
t.Fatalf("repeat decision changed the run: %+v", repeat)
}
if _, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "go"}); err == nil || !strings.Contains(err.Error(), "not waiting") {
t.Fatalf("conflicting decision on a finished run did not report not waiting: %v", err)
}
r.mu.Lock()
delete(r.active, run.ID)
r.mu.Unlock()
}
+20 -2
View File
@@ -87,6 +87,15 @@ type runner struct {
}
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
return newRunnerAt(ctx, s, "")
}
// newRunnerAt builds a runner whose stage artifacts live under artifactsRoot
// ("" means <store directory>/artifacts). artifactsRoot is fixed before any
// background goroutine (session export jobs, notifications, the size backfill)
// starts and is never written afterwards, so those goroutines read it without
// locking.
func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (*runner, error) {
if err := s.AcquireRunnerLock(); err != nil {
return nil, err
}
@@ -123,7 +132,10 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
if err := s.RecoverRuns(ctx); err != nil {
return nil, err
}
root := filepath.Join(s.Directory(), "artifacts")
root := artifactsRoot
if root == "" {
root = filepath.Join(s.Directory(), "artifacts")
}
active := map[int64]activeRun{}
rows, err := s.ListRuns(ctx)
if err != nil {
@@ -780,7 +792,13 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
r.mu.Unlock()
return nil, err
}
if _, busy := r.active[detail.ID]; busy {
// finishRunState persists the terminal status under r.mu, but the run stays
// in r.active until its goroutine finishes notifying and the launch teardown
// runs. A run whose terminal state is already durable (FinishedAt is only set
// once cleanup is confirmed) has no executor left to race with a decision, so
// ResolveDecision answers it ("already decided" or "not waiting") and never
// resumes a terminal run. A run that is still executing is refused.
if _, busy := r.active[detail.ID]; busy && detail.FinishedAt == "" {
r.mu.Unlock()
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
}
@@ -112,12 +112,11 @@ func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) {
t.Fatalf("fixture is not a parked export: %+v", parked)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunner(ctx, reopened)
r, err := newRunnerAt(ctx, reopened, root)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
r.artifactsRoot = root
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "two" {
t.Errorf("replayed owner: %s", def.Key)
+2 -4
View File
@@ -27,9 +27,8 @@ func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRe
if err != nil {
t.Fatal(err)
}
h := Handler(ctx, s).(*runHandler)
h := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, s).(*runHandler)
t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() })
h.runner.artifactsRoot = t.TempDir()
h.runner.exportSession = export
if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
@@ -352,12 +351,11 @@ func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunner(ctx, reopened)
r, err := newRunnerAt(ctx, reopened, root)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
r.artifactsRoot = root
r.exportSession = validGateExport
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "two" {
+1 -2
View File
@@ -183,12 +183,11 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sessionExportCLI(t) // read-only native CLI fixture, never a live model/run
r, err := newRunner(ctx, s)
r, err := newRunnerAt(ctx, s, root)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }()
r.artifactsRoot = root
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
t.Error("restart recovery reran a stage")
return stage.Result{}, errors.New("forbidden rerun")
+1 -2
View File
@@ -66,8 +66,7 @@ func TestSessionExportLive(t *testing.T) {
if err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store).(*runHandler)
handler.runner.artifactsRoot = filepath.Join(root, "artifacts")
handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: filepath.Join(root, "artifacts")}, store).(*runHandler)
defer func() { cancel(); WaitRuns(handler) }()
workspace := filepath.Join(root, "ws")
if err := os.Mkdir(workspace, 0700); err != nil {
+1 -2
View File
@@ -95,9 +95,8 @@ func sessionExportRunner(t *testing.T, backend string, seam func(stage.SessionEx
if err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store).(*runHandler)
handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, store).(*runHandler)
t.Cleanup(func() { cancel(); WaitRuns(handler) })
handler.runner.artifactsRoot = t.TempDir()
if seam != nil {
handler.runner.exportSession = func(_ context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) {
return seam(request)
+1 -4
View File
@@ -77,13 +77,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
var runs *runner
var runErr error
if len(stores) > 0 {
runs, runErr = newRunner(ctx, stores[0])
runs, runErr = newRunnerAt(ctx, stores[0], options.ArtifactsRoot)
if runErr == nil {
runs.events = events
}
if runErr == nil && options.ArtifactsRoot != "" {
runs.artifactsRoot = options.ArtifactsRoot
}
if runErr == nil && options.RunTitles != nil {
runs.title = options.RunTitles
}