fix: park human runs atomically before slow notification work
This commit is contained in:
@@ -155,7 +155,7 @@ func TestSessionExportJobGuardsCleanupAgentsAndExactOwnership(t *testing.T) {
|
||||
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.db.Exec("ALTER TABLE pipeline_runs ADD COLUMN cleanup_unconfirmed INTEGER NOT NULL DEFAULT 0"); err != nil {
|
||||
if err := s.addColumnIfMissing("pipeline_runs", "cleanup_unconfirmed", "INTEGER NOT NULL DEFAULT 0"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, queries := range [][2]string{
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
)
|
||||
|
||||
func TestParkedLaunchCannotRemoveResumedExecution(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "generation.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
r := &runner{ctx: ctx, store: store, active: map[int64]activeRun{}}
|
||||
firstCtx := make(chan context.Context, 1)
|
||||
firstRelease := make(chan struct{})
|
||||
r.mu.Lock()
|
||||
r.launch(1, "/work", func(ctx context.Context) { firstCtx <- ctx; <-firstRelease })
|
||||
r.mu.Unlock()
|
||||
old := <-firstCtx
|
||||
r.mu.Lock()
|
||||
delete(r.active, 1) // human-stage park transition
|
||||
r.launch(1, "/work", func(ctx context.Context) { <-ctx.Done() })
|
||||
resumed := r.active[1]
|
||||
r.mu.Unlock()
|
||||
close(firstRelease)
|
||||
<-old.Done() // registry cleanup precedes launch's deferred cancel
|
||||
r.mu.Lock()
|
||||
current, ok := r.active[1]
|
||||
r.mu.Unlock()
|
||||
if !ok || current.generation != resumed.generation {
|
||||
t.Fatal("parked launch removed the resumed execution reservation")
|
||||
}
|
||||
cancel()
|
||||
r.wg.Wait()
|
||||
}
|
||||
@@ -27,8 +27,9 @@ import (
|
||||
// and resolved workspace reservation. A single map is the active-run state, so the
|
||||
// workspace cannot drift from the cancel function.
|
||||
type activeRun struct {
|
||||
cancel context.CancelFunc
|
||||
workspace string
|
||||
cancel context.CancelFunc
|
||||
workspace string
|
||||
generation *byte
|
||||
}
|
||||
|
||||
type runner struct {
|
||||
@@ -348,7 +349,8 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
|
||||
// through it, so registration and teardown live once. Callers must hold r.mu.
|
||||
func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
|
||||
ctx, cancel := context.WithCancel(r.ctx)
|
||||
r.active[id] = activeRun{cancel: cancel, workspace: workspace}
|
||||
generation := new(byte)
|
||||
r.active[id] = activeRun{cancel: cancel, workspace: workspace, generation: generation}
|
||||
r.wg.Add(1)
|
||||
go func() {
|
||||
defer r.wg.Done()
|
||||
@@ -356,6 +358,9 @@ func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
|
||||
defer func() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if entry, ok := r.active[id]; !ok || entry.generation != generation {
|
||||
return // A parked launch must not remove the resumed reservation.
|
||||
}
|
||||
check, done := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer done()
|
||||
detail, err := r.store.GetRun(check, id)
|
||||
@@ -1587,7 +1592,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
// A human step is O(1): one BeginStep plus one MarkWaiting, no
|
||||
// filediff capture, no artifacts directory and no workspace mutation.
|
||||
evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges))
|
||||
r.mu.Lock()
|
||||
if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil {
|
||||
r.mu.Unlock()
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
return
|
||||
@@ -1606,6 +1613,10 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
CompletedSteps: state.completed,
|
||||
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting},
|
||||
}, false)
|
||||
// Publishing can be slow. Once waiting is visible, release execution
|
||||
// atomically so a decision is not spuriously rejected as "active".
|
||||
delete(r.active, id)
|
||||
r.mu.Unlock()
|
||||
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
|
||||
r.deliverRunNotificationBatch(time.Now(), 1)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user