Fix decide-after-finish refusal and run size loop data race
A run stays in r.active after finishRunState persists its terminal status until its goroutine finishes notifying, so a decision in that window was refused as "already active". decide now refuses only runs whose terminal state is not yet durable; a finished run gets ResolveDecision's already-decided / not-waiting answer, and an executing run is still refused. Tests overwrote runner.artifactsRoot after Handler/newRunner had started background goroutines (sizeLoop, session export jobs) that read it. The root is now fixed at construction through newRunnerAt, which HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it there instead of mutating the field afterwards.
This commit is contained in:
@@ -215,9 +215,8 @@ func runWithArtifact(t *testing.T) (*runHandler, protocol.RunStep) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store).(*runHandler)
|
||||
handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, store).(*runHandler)
|
||||
t.Cleanup(func() { cancel(); WaitRuns(handler) })
|
||||
handler.runner.artifactsRoot = t.TempDir()
|
||||
handler.runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
if def.Turn.Private == "" || def.Turn.Public == "" {
|
||||
t.Error("runner did not provide the turn artifact scopes")
|
||||
|
||||
@@ -154,12 +154,11 @@ func TestOutputFormatRetryInheritanceThroughRuntime(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := s.List(ctx)
|
||||
r, err := newRunner(ctx, s)
|
||||
r, err := newRunnerAt(ctx, s, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.wg.Wait() }()
|
||||
r.artifactsRoot = t.TempDir()
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{}, fmt.Errorf("fixture failure")
|
||||
}
|
||||
|
||||
@@ -431,3 +431,63 @@ func TestHumanStageWaitingNotification(t *testing.T) {
|
||||
t.Fatalf("waiting notification receipt missing: %+v", detail.Steps[0].Notifications)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecideOnFinishingRunIsAnsweredNotRefused pins the window between
|
||||
// finishRunState persisting a terminal status and the launch teardown removing
|
||||
// the run from r.active: a decision then gets the store's idempotent/not-waiting
|
||||
// answer, while a run that is still executing is refused as active.
|
||||
func TestDecideOnFinishingRunIsAnsweredNotRefused(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
r, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.wg.Wait()
|
||||
r.execute = successExecute
|
||||
run := startHumanRun(t, ctx, r, store, "human-finishing")
|
||||
|
||||
// A waiting run that is genuinely executing (registered, not finished) is
|
||||
// refused.
|
||||
r.mu.Lock()
|
||||
r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace}
|
||||
r.mu.Unlock()
|
||||
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt"}); err == nil || !strings.Contains(err.Error(), "already active") {
|
||||
t.Fatalf("decision on an executing run was not refused: %v", err)
|
||||
}
|
||||
r.mu.Lock()
|
||||
delete(r.active, run.ID)
|
||||
r.mu.Unlock()
|
||||
|
||||
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt", Feedback: "stop"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
if detail, err := store.GetRun(ctx, run.ID); err != nil || detail.Status != "succeeded" {
|
||||
t.Fatalf("run did not finish: %+v %v", detail, err)
|
||||
}
|
||||
|
||||
// Simulate the finishing window: terminal status persisted, reservation not
|
||||
// yet released.
|
||||
r.mu.Lock()
|
||||
r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace}
|
||||
r.mu.Unlock()
|
||||
repeat, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "halt"})
|
||||
if err != nil {
|
||||
t.Fatalf("repeat decision on a finished run was refused: %v", err)
|
||||
}
|
||||
if repeat.Status != "succeeded" || repeat.Steps[0].Output["feedback"] != "stop" {
|
||||
t.Fatalf("repeat decision changed the run: %+v", repeat)
|
||||
}
|
||||
if _, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "go"}); err == nil || !strings.Contains(err.Error(), "not waiting") {
|
||||
t.Fatalf("conflicting decision on a finished run did not report not waiting: %v", err)
|
||||
}
|
||||
r.mu.Lock()
|
||||
delete(r.active, run.ID)
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -87,6 +87,15 @@ type runner struct {
|
||||
}
|
||||
|
||||
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
return newRunnerAt(ctx, s, "")
|
||||
}
|
||||
|
||||
// newRunnerAt builds a runner whose stage artifacts live under artifactsRoot
|
||||
// ("" means <store directory>/artifacts). artifactsRoot is fixed before any
|
||||
// background goroutine (session export jobs, notifications, the size backfill)
|
||||
// starts and is never written afterwards, so those goroutines read it without
|
||||
// locking.
|
||||
func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (*runner, error) {
|
||||
if err := s.AcquireRunnerLock(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -123,7 +132,10 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
if err := s.RecoverRuns(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root := filepath.Join(s.Directory(), "artifacts")
|
||||
root := artifactsRoot
|
||||
if root == "" {
|
||||
root = filepath.Join(s.Directory(), "artifacts")
|
||||
}
|
||||
active := map[int64]activeRun{}
|
||||
rows, err := s.ListRuns(ctx)
|
||||
if err != nil {
|
||||
@@ -780,7 +792,13 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
r.mu.Unlock()
|
||||
return nil, err
|
||||
}
|
||||
if _, busy := r.active[detail.ID]; busy {
|
||||
// finishRunState persists the terminal status under r.mu, but the run stays
|
||||
// in r.active until its goroutine finishes notifying and the launch teardown
|
||||
// runs. A run whose terminal state is already durable (FinishedAt is only set
|
||||
// once cleanup is confirmed) has no executor left to race with a decision, so
|
||||
// ResolveDecision answers it ("already decided" or "not waiting") and never
|
||||
// resumes a terminal run. A run that is still executing is refused.
|
||||
if _, busy := r.active[detail.ID]; busy && detail.FinishedAt == "" {
|
||||
r.mu.Unlock()
|
||||
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
|
||||
}
|
||||
|
||||
@@ -112,12 +112,11 @@ func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) {
|
||||
t.Fatalf("fixture is not a parked export: %+v", parked)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
r, err := newRunner(ctx, reopened)
|
||||
r, err := newRunnerAt(ctx, reopened, root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
|
||||
r.artifactsRoot = root
|
||||
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
if def.Key != "two" {
|
||||
t.Errorf("replayed owner: %s", def.Key)
|
||||
|
||||
@@ -27,9 +27,8 @@ func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRe
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := Handler(ctx, s).(*runHandler)
|
||||
h := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, s).(*runHandler)
|
||||
t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() })
|
||||
h.runner.artifactsRoot = t.TempDir()
|
||||
h.runner.exportSession = export
|
||||
if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -352,12 +351,11 @@ func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
r, err := newRunner(ctx, reopened)
|
||||
r, err := newRunnerAt(ctx, reopened, root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
|
||||
r.artifactsRoot = root
|
||||
r.exportSession = validGateExport
|
||||
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
if def.Key != "two" {
|
||||
|
||||
@@ -183,12 +183,11 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
sessionExportCLI(t) // read-only native CLI fixture, never a live model/run
|
||||
r, err := newRunner(ctx, s)
|
||||
r, err := newRunnerAt(ctx, s, root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }()
|
||||
r.artifactsRoot = root
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
t.Error("restart recovery reran a stage")
|
||||
return stage.Result{}, errors.New("forbidden rerun")
|
||||
|
||||
@@ -66,8 +66,7 @@ func TestSessionExportLive(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store).(*runHandler)
|
||||
handler.runner.artifactsRoot = filepath.Join(root, "artifacts")
|
||||
handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: filepath.Join(root, "artifacts")}, store).(*runHandler)
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
workspace := filepath.Join(root, "ws")
|
||||
if err := os.Mkdir(workspace, 0700); err != nil {
|
||||
|
||||
@@ -95,9 +95,8 @@ func sessionExportRunner(t *testing.T, backend string, seam func(stage.SessionEx
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store).(*runHandler)
|
||||
handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, store).(*runHandler)
|
||||
t.Cleanup(func() { cancel(); WaitRuns(handler) })
|
||||
handler.runner.artifactsRoot = t.TempDir()
|
||||
if seam != nil {
|
||||
handler.runner.exportSession = func(_ context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
return seam(request)
|
||||
|
||||
@@ -77,13 +77,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
var runs *runner
|
||||
var runErr error
|
||||
if len(stores) > 0 {
|
||||
runs, runErr = newRunner(ctx, stores[0])
|
||||
runs, runErr = newRunnerAt(ctx, stores[0], options.ArtifactsRoot)
|
||||
if runErr == nil {
|
||||
runs.events = events
|
||||
}
|
||||
if runErr == nil && options.ArtifactsRoot != "" {
|
||||
runs.artifactsRoot = options.ArtifactsRoot
|
||||
}
|
||||
if runErr == nil && options.RunTitles != nil {
|
||||
runs.title = options.RunTitles
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user