From b96b9d94a6b8bf6958e95a2a38609b0466879851 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Tue, 20 Oct 2026 15:47:07 +0100 Subject: [PATCH] fix: keep cancellation pending until execution cleanup is confirmed --- cmd/pipelines-server/main.go | 3 + docs/cli.md | 24 +++- internal/pipeline/cancel_recovery_test.go | 70 ++++++++++ internal/pipeline/human_test.go | 16 ++- internal/pipeline/runner_lock.go | 54 ++++++++ internal/pipeline/runner_lock_test.go | 60 ++++++++ internal/pipeline/runs.go | 4 +- internal/pipeline/runs_mutate.go | 61 ++++---- internal/pipeline/runs_schema.go | 9 +- internal/pipeline/runs_sessions.go | 24 +++- internal/pipeline/store.go | 21 ++- internal/protocol/runevents.go | 22 +-- internal/protocol/runs.go | 28 ++-- internal/server/concurrency_test.go | 9 +- internal/server/execution_safety_test.go | 153 ++++++++++++++++++++ internal/server/file_changes_test.go | 6 + internal/server/runevents.go | 30 ++-- internal/server/runner.go | 162 ++++++++++++++++------ internal/server/runner_test.go | 14 +- internal/server/session_export_test.go | 2 +- web/src/actions.js | 2 +- web/src/runEvents.js | 12 +- web/src/runs.js | 8 ++ web/src/views/runDetail.js | 6 +- web/src/views/runs.js | 6 +- web/test/executionSafety.test.mjs | 26 ++++ 26 files changed, 690 insertions(+), 142 deletions(-) create mode 100644 internal/pipeline/cancel_recovery_test.go create mode 100644 internal/pipeline/runner_lock.go create mode 100644 internal/pipeline/runner_lock_test.go create mode 100644 internal/server/execution_safety_test.go create mode 100644 web/test/executionSafety.test.mjs diff --git a/cmd/pipelines-server/main.go b/cmd/pipelines-server/main.go index fec2be21..d6c44b46 100644 --- a/cmd/pipelines-server/main.go +++ b/cmd/pipelines-server/main.go @@ -42,6 +42,9 @@ func main() { log.Fatal(err) } defer store.Close() + if err := store.AcquireServerLock(); err != nil { + log.Fatal(err) + } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() artifactsRoot := filepath.Join(filepath.Dir(*db), "artifacts") diff --git a/docs/cli.md b/docs/cli.md index a147e952..ff9b4eb2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -166,13 +166,35 @@ Every submission, including agent-issued `submit`/`rpc`, still uses server valid ## Drafts in the TUI -The server runs one active run per resolved workspace. A new request key in a +The server runs one active run per resolved workspace, including overlapping +parent/descendant directories (and symlink aliases). Separate sibling worktrees +remain independent. A new request key in a different workspace is admitted immediately and executes concurrently; a second run in a workspace an active run is already changing is refused, naming the conflicting run and advising you to wait or use a different workspace (for example another checkout of the repository). Reusing the original request key reattaches idempotently. +`cancel ID` / `cancel_run` records `cancel_requested: true` before signaling the +executor. The reply can still have `status: running` (or `queued`) and no +`finished_at`: this means **stopping**, not stopped. The workspace stays reserved +until execution returns and owned-session cleanup is confirmed; late success +cannot override cancellation. A parked human run has no executor and cancels +immediately. Repeating Stop while stopping is safe. + +If cleanup cannot be confirmed, the run stays nonterminal with +`cleanup_unconfirmed: true`, retains partial step evidence, and reserves its +workspace. The web client labels this explicitly and hides duplicate Stop. +Successful startup recovery interrupts and confirms recorded sessions before +settling the run; without a recoverable session pointer, uncertainty and the +reservation survive restart rather than inventing stop confirmation. Recovery +does not resume a canceled run. A database-sidecar advisory server lock is held +through runner shutdown, so a second server on that database cannot recover or +interrupt the first one's live runs. Keep the sidecar file in place; its existence +is not ownership (the kernel lock is). This is a local-filesystem lock, not a +distributed lease; hard-link database aliases and filesystems without reliable +advisory locking are unsupported. + Use `:templates` for four simple, manual-only starters: task, review, research, and plan/implement/review. **Submit template** (`s` or Enter) asks for a name without opening an editor. Enter saves a retained draft, then a separate diff --git a/internal/pipeline/cancel_recovery_test.go b/internal/pipeline/cancel_recovery_test.go new file mode 100644 index 00000000..81e95383 --- /dev/null +++ b/internal/pipeline/cancel_recovery_test.go @@ -0,0 +1,70 @@ +package pipeline + +import ( + "context" + "path/filepath" + "testing" +) + +func TestCancelRecoveryRequiresCleanupConfirmation(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "recovery.db") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer func() { store.Close() }() + if _, err := store.Submit(ctx, "recovery", Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, _ := store.List(ctx) + definition, _ := store.Get(ctx, items[0].ID, 1) + id, _, err := store.CreateRun(ctx, definition, "recover", t.TempDir(), false, nil, map[string]any{"prompt": "task"}) + if err != nil { + t.Fatal(err) + } + if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil { + t.Fatal(err) + } + if err := store.RunSession(ctx, id, 1, "ses_cleanup", true, "v2", "fixture"); err != nil { + t.Fatal(err) + } + if err := store.Cancel(ctx, id); err != nil { + t.Fatal(err) + } + if err := store.MarkCleanupUnconfirmed(ctx, id); err != nil { + t.Fatal(err) + } + if err := store.FinishRun(ctx, id, "canceled", "", "", nil); err == nil { + t.Fatal("unconfirmed cleanup accepted terminal state") + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + store, err = Open(path) + if err != nil { + t.Fatal(err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + detail, _ := store.GetRun(ctx, id) + if detail.Status != "running" || !detail.CancelRequested || !detail.CleanupUnconfirmed || !detail.Steps[0].AgentActive || detail.Steps[0].FinishedAt != "" { + t.Fatal("recovery falsely settled unconfirmed execution", detail) + } + owners, err := store.ActiveSessionOwners(ctx) + if err != nil || len(owners) != 1 { + t.Fatalf("lost recovery owner: %+v %v", owners, err) + } + // The runner calls this only after InterruptAndConfirm succeeds. + if err := store.ConfirmRecoveredSession(ctx, owners[0]); err != nil { + t.Fatal(err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + detail, _ = store.GetRun(ctx, id) + if detail.Status != "canceled" || detail.CleanupUnconfirmed || detail.FinishedAt == "" || detail.Steps[0].Status != "canceled" || detail.Steps[0].AgentActive { + t.Fatal("confirmed recovery failed to settle cancellation", detail) + } +} diff --git a/internal/pipeline/human_test.go b/internal/pipeline/human_test.go index 24672851..58712f94 100644 --- a/internal/pipeline/human_test.go +++ b/internal/pipeline/human_test.go @@ -183,7 +183,7 @@ func TestResolveDecisionValidUnknownRepeatAndNotWaiting(t *testing.T) { } } -func TestCancelRunningRunMarksRunAndLiveStep(t *testing.T) { +func TestCancelRunningRunRecordsIntentUntilExecutionReturns(t *testing.T) { ctx := context.Background() store, err := Open(filepath.Join(t.TempDir(), "cancel-running.db")) if err != nil { @@ -212,8 +212,18 @@ func TestCancelRunningRunMarksRunAndLiveStep(t *testing.T) { if err != nil { t.Fatal(err) } - if detail.Status != "canceled" || detail.Steps[0].Status != "canceled" { - t.Fatalf("running run not canceled immediately: %+v %+v", detail.RunSummary, detail.Steps) + if detail.Status != "running" || detail.Steps[0].Status != "running" || !detail.CancelRequested || detail.FinishedAt != "" { + t.Fatalf("stop request falsely settled execution: %+v %+v", detail.RunSummary, detail.Steps) + } + if err := store.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "gate", Status: "completed", Outcome: "success"}); err != nil { + t.Fatal(err) + } + if err := store.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil { + t.Fatal(err) + } + detail, err = store.GetRun(ctx, id) + if err != nil || detail.Status != "canceled" || detail.Steps[0].Status != "canceled" { + t.Fatalf("late success beat cancellation: %+v %v", detail, err) } if err := store.Cancel(ctx, id); err == nil { t.Fatal("canceling an already-canceled run succeeded") diff --git a/internal/pipeline/runner_lock.go b/internal/pipeline/runner_lock.go new file mode 100644 index 00000000..acce0cae --- /dev/null +++ b/internal/pipeline/runner_lock.go @@ -0,0 +1,54 @@ +package pipeline + +import ( + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +// AcquireRunnerLock must precede recovery: a second server must never interrupt +// the first server's live sessions. The kernel releases the advisory lock on +// process exit; the sidecar is deliberately never unlinked (avoiding inode races). +// The store must stay open until all runner goroutines have stopped. +func (s *Store) AcquireRunnerLock() error { + s.mu.Lock() + defer s.mu.Unlock() + if s.runnerClaimed { + return fmt.Errorf("a runner already owns this store") + } + if err := s.acquireServerLock(); err != nil { + return err + } + s.runnerClaimed = true + return nil +} + +// AcquireServerLock lets the executable fail before attaching chat or serving +// requests. newRunner also claims it, protecting embedded handler callers. +func (s *Store) AcquireServerLock() error { + s.mu.Lock() + defer s.mu.Unlock() + return s.acquireServerLock() +} + +func (s *Store) acquireServerLock() error { + if s.runnerLock != nil { + return nil + } + path, err := filepath.EvalSymlinks(s.path) + if err != nil { + return fmt.Errorf("resolve server database: %w", err) + } + file, err := os.OpenFile(path+".server.lock", os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return err + } + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + file.Close() + return fmt.Errorf("another server owns database %s: %w", path, err) + } + s.runnerLock = file + return nil +} diff --git a/internal/pipeline/runner_lock_test.go b/internal/pipeline/runner_lock_test.go new file mode 100644 index 00000000..85fa0c9c --- /dev/null +++ b/internal/pipeline/runner_lock_test.go @@ -0,0 +1,60 @@ +package pipeline + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestRunnerLockCrossProcessAndRelease(t *testing.T) { + if path := os.Getenv("PIPELINES_TEST_LOCK_DB"); path != "" { + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if err := store.AcquireRunnerLock(); err == nil { + t.Fatal("child acquired parent's server lock") + } + return + } + path := filepath.Join(t.TempDir(), "lock.db") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if err := store.AcquireServerLock(); err != nil { + t.Fatal(err) + } + if err := store.AcquireRunnerLock(); err != nil { + t.Fatal(err) + } + if err := store.AcquireRunnerLock(); err == nil { + t.Fatal("two runners claimed one store") + } + cmd := exec.Command(os.Args[0], "-test.run=^TestRunnerLockCrossProcessAndRelease$") + cmd.Env = append(os.Environ(), "PIPELINES_TEST_LOCK_DB="+path) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("cross-process lock: %v\n%s", err, output) + } + alias := filepath.Join(t.TempDir(), "alias.db") + if err := os.Symlink(path, alias); err != nil { + t.Fatal(err) + } + other, err := Open(alias) + if err != nil { + t.Fatal(err) + } + defer other.Close() + if err := other.AcquireRunnerLock(); err == nil { + t.Fatal("symlink bypassed lock") + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + if err := other.AcquireRunnerLock(); err != nil { + t.Fatal("lock not released with store", err) + } +} diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index fd396949..6105c097 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -21,12 +21,12 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage, ` + runStamps + `,r.diagnostic, CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END, -(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode +(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed ` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version` func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error { var output, stageModels string - if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode); err != nil { + if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed); err != nil { return err } if stageModels != "" && stageModels != "{}" { diff --git a/internal/pipeline/runs_mutate.go b/internal/pipeline/runs_mutate.go index 8a0536f2..8bc44137 100644 --- a/internal/pipeline/runs_mutate.go +++ b/internal/pipeline/runs_mutate.go @@ -44,7 +44,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin return err } defer tx.Rollback() - if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running')", key, id)); err != nil { + if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0", key, id)); err != nil { return err } if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil { @@ -86,8 +86,16 @@ func (s *Store) FinishStep(ctx context.Context, id int64, step protocol.RunStep) return err } defer tx.Rollback() - if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,provider=?,model=?,finished_at=CURRENT_TIMESTAMP -WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, step.Provider, step.Model, id, step.Sequence)); err != nil { + var requested bool + if err := tx.QueryRowContext(ctx, "SELECT cancel_requested FROM pipeline_runs WHERE id=?", id).Scan(&requested); err != nil { + return err + } + if requested && step.Status != "running" { + step.Status, step.Outcome, step.Target = "canceled", "", "" + step.Diagnostic = "Canceled by operator; execution returned" + } + if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,provider=?,model=?,finished_at=CASE WHEN ?='running' THEN NULL ELSE CURRENT_TIMESTAMP END +WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, step.Provider, step.Model, step.Status, id, step.Sequence)); err != nil { return err } if step.FileChanges != nil { @@ -288,7 +296,7 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error { return err } defer tx.Rollback() - res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',finished_at=CURRENT_TIMESTAMP WHERE id=? AND status=?`, id, protocol.RunStatusWaiting) + res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,diagnostic='Canceled while waiting for a decision',finished_at=CURRENT_TIMESTAMP WHERE id=? AND status=?`, id, protocol.RunStatusWaiting) if err != nil { return err } @@ -305,35 +313,11 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error { return tx.Commit() } -// Cancel immediately marks an active run and its live step canceled. The -// runner's context is canceled too, but a harness that ignores cancellation (or -// is stuck awaiting a provider) must not leave the operator looking at a run -// that still says "running" after they pressed Stop. +// Cancel records intent only; it is not evidence that the executor stopped. func (s *Store) Cancel(ctx context.Context, id int64) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() - res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs -SET status='canceled', diagnostic=CASE WHEN diagnostic='' THEN 'Canceled by operator' ELSE diagnostic END, finished_at=CURRENT_TIMESTAMP -WHERE id=? AND status IN ('queued','running','waiting')`, id) - if err != nil { - return err - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n != 1 { - return fmt.Errorf("run is not active or waiting") - } - if _, err := tx.ExecContext(ctx, `UPDATE run_steps -SET status='canceled', diagnostic=CASE WHEN diagnostic='' THEN 'Canceled by operator' ELSE diagnostic END, finished_at=CURRENT_TIMESTAMP -WHERE run_id=? AND status IN ('running','waiting')`, id); err != nil { - return err - } - return tx.Commit() + return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cancel_requested=1, +diagnostic=CASE WHEN cleanup_unconfirmed=1 THEN diagnostic ELSE 'Stop requested; waiting for execution and cleanup to stop' END +WHERE id=? AND status IN ('queued','running','waiting')`, id)) } func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagnostic string, output map[string]any) error { @@ -341,5 +325,16 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno if err != nil { return err } - return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id)) + return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET +status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END, +outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END, +diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END, +output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0 +AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id)) +} + +func (s *Store) MarkCleanupUnconfirmed(ctx context.Context, id int64) error { + return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cleanup_unconfirmed=1, +diagnostic='Agent cleanup was not confirmed; workspace remains reserved until successful restart recovery' +WHERE id=? AND status IN ('queued','running')`, id)) } diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index 1d572f62..9e1f6ce1 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -62,6 +62,11 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints ( if err := s.migrateRunTitles(); err != nil { return err } + for _, column := range []string{"cancel_requested", "cleanup_unconfirmed"} { + if err := s.addColumnIfMissing("pipeline_runs", column, "INTEGER NOT NULL DEFAULT 0"); err != nil { + return err + } + } if err := s.addColumnIfMissing("pipeline_runs", "notify_mode", "TEXT NOT NULL DEFAULT 'auto'"); err != nil { return err } @@ -184,10 +189,10 @@ func (s *Store) RecoverRuns(ctx context.Context) error { defer tx.Rollback() // No replay: pending visits never began. A cleanup agent that was still // active is marked interrupted, and its run is interrupted below. - if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil { + if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)", protocol.StepStatusWaiting); err != nil { return err } - if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic='Server stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>?", protocol.RunStatusWaiting); err != nil { + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil { return err } return tx.Commit() diff --git a/internal/pipeline/runs_sessions.go b/internal/pipeline/runs_sessions.go index 163cd303..f566416f 100644 --- a/internal/pipeline/runs_sessions.go +++ b/internal/pipeline/runs_sessions.go @@ -32,8 +32,18 @@ func (s *Store) RunSession(ctx context.Context, id int64, sequence int, session // ConfirmRecoveredSession clears the agent-active marker after startup has // interrupted a previously active session, so it no longer blocks new work. func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error { - _, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID) - return err + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET cleanup_unconfirmed=0 WHERE id=? AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=? AND agent_active=1)`, owner.RunID, owner.RunID); err != nil { + return err + } + return tx.Commit() } type SessionOwner struct { @@ -83,8 +93,14 @@ FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id LEFT JOIN run_session_own if activeOnly { query += " AND s.agent_active=1" } - query += " ORDER BY s.run_id DESC LIMIT ?" - rows, err := s.db.QueryContext(ctx, query, sessionOwnerScanLimit) + query += " ORDER BY s.run_id DESC" + var args []any + if !activeOnly { + query += " LIMIT ?" + args = append(args, sessionOwnerScanLimit) + } + // Recovery cannot silently drop old but still-active owners at the history cap. + rows, err := s.db.QueryContext(ctx, query, args...) if err != nil { return nil, err } diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index b7c1e34c..542c9ec6 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -18,10 +18,12 @@ import ( ) type Store struct { - db *sql.DB - dir string - path string - mu sync.Mutex + db *sql.DB + dir string + path string + mu sync.Mutex + runnerLock *os.File + runnerClaimed bool } // openDB opens one SQLite connection pool for a DSN. Every open path shares it @@ -73,7 +75,16 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU return s, nil } -func (s *Store) Close() error { return s.db.Close() } +func (s *Store) Close() error { + err := s.db.Close() + s.mu.Lock() + defer s.mu.Unlock() + if s.runnerLock != nil { + _ = s.runnerLock.Close() + s.runnerLock = nil + } + return err +} // OpenReadOnly opens an existing database without running schema DDL or // changing journal mode, so a live database can be queried read-only while its diff --git a/internal/protocol/runevents.go b/internal/protocol/runevents.go index 374f6cad..49dac46c 100644 --- a/internal/protocol/runevents.go +++ b/internal/protocol/runevents.go @@ -29,16 +29,18 @@ type RunEventFrame struct { Workspace string `json:"workspace,omitempty"` // PipelineID, Name, Title and Version identify the run's pipeline and its // operator-facing title, so a new run's row can be built without a fetch. - PipelineID int64 `json:"pipeline_id,omitempty"` - Name string `json:"name,omitempty"` - Title string `json:"title,omitempty"` - Version int `json:"version,omitempty"` - Status string `json:"status,omitempty"` - Outcome string `json:"outcome,omitempty"` - CurrentStage string `json:"current_stage,omitempty"` - CompletedSteps int `json:"completed_steps,omitempty"` - CreatedAt string `json:"created_at,omitempty"` - FinishedAt string `json:"finished_at,omitempty"` + PipelineID int64 `json:"pipeline_id,omitempty"` + Name string `json:"name,omitempty"` + Title string `json:"title,omitempty"` + Version int `json:"version,omitempty"` + Status string `json:"status,omitempty"` + CancelRequested bool `json:"cancel_requested,omitempty"` + CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"` + Outcome string `json:"outcome,omitempty"` + CurrentStage string `json:"current_stage,omitempty"` + CompletedSteps int `json:"completed_steps,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + FinishedAt string `json:"finished_at,omitempty"` // Artifacts is the number of files recorded for the referenced step, so a // client can surface artifact capture without reading the step. Artifacts int `json:"artifacts,omitempty"` diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 68da68bb..15ae1f72 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -126,19 +126,21 @@ type RunSummary struct { // Title is the operator-facing, bounded single-line description of what the // run is for, generated from its prompt. It is never empty for a new run; // historical rows predating the title column may omit it. - Title string `json:"title,omitempty"` - Version int `json:"version"` - ContentHash string `json:"content_hash"` - Status string `json:"status"` - Executing bool `json:"executing"` - Outcome string `json:"outcome"` - CurrentStage string `json:"current_stage"` - Workspace string `json:"workspace,omitempty"` - CreatedAt string `json:"created_at"` - FinishedAt string `json:"finished_at"` - Diagnostic string `json:"diagnostic"` - ResultPreview string `json:"result_preview,omitempty"` - CompletedSteps int `json:"completed_steps"` + Title string `json:"title,omitempty"` + Version int `json:"version"` + ContentHash string `json:"content_hash"` + Status string `json:"status"` + CancelRequested bool `json:"cancel_requested,omitempty"` + CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"` + Executing bool `json:"executing"` + Outcome string `json:"outcome"` + CurrentStage string `json:"current_stage"` + Workspace string `json:"workspace,omitempty"` + CreatedAt string `json:"created_at"` + FinishedAt string `json:"finished_at"` + Diagnostic string `json:"diagnostic"` + ResultPreview string `json:"result_preview,omitempty"` + CompletedSteps int `json:"completed_steps"` // Model and StageModels are the launch-time overrides this run records so it // stays reproducible: Model covers every model stage and StageModels overrides // individual stages by key. They are empty for a run that used the pinned diff --git a/internal/server/concurrency_test.go b/internal/server/concurrency_test.go index a60bf7e7..2b6b335b 100644 --- a/internal/server/concurrency_test.go +++ b/internal/server/concurrency_test.go @@ -514,9 +514,8 @@ func waitEnteredString(t *testing.T, entered <-chan string) { } } -// TestOwnActiveSessionStillFailsRun proves a run still fails when one of its OWN -// agent sessions is left active after the stage completes. -func TestOwnActiveSessionStillFailsRun(t *testing.T) { +// Active ownership is not a terminal failure: the agent might still mutate files. +func TestOwnActiveSessionKeepsRunUnsettled(t *testing.T) { _, r, pipelineID := agentStore(t) workspace := t.TempDir() ready := make(chan struct{}) @@ -539,7 +538,7 @@ func TestOwnActiveSessionStillFailsRun(t *testing.T) { if err != nil { t.Fatal(err) } - if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "agent quiescence not confirmed") { - t.Fatalf("own active session did not fail the run: %+v", detail) + if detail.Status != "running" || !detail.CleanupUnconfirmed || detail.FinishedAt != "" { + t.Fatalf("own active session falsely settled the run: %+v", detail) } } diff --git a/internal/server/execution_safety_test.go b/internal/server/execution_safety_test.go new file mode 100644 index 00000000..f1b5840a --- /dev/null +++ b/internal/server/execution_safety_test.go @@ -0,0 +1,153 @@ +package server + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestCancelWaitsForContextIgnoringExecutor(t *testing.T) { + ctx, r, pipelineID := concurrentStore(t) + entered, release := make(chan struct{}, 1), make(chan struct{}) + var releaseOnce sync.Once + stopExecutor := func() { releaseOnce.Do(func() { close(release) }) } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + entered <- struct{}{} + <-release // Deliberately ignore the canceled context. + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + defer func() { stopExecutor(); r.wg.Wait() }() + workspace := t.TempDir() + request := protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "blocking", Workspace: workspace, Input: map[string]any{"prompt": "task"}} + run, err := r.start(request) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + if err := r.cancel(run.ID); err != nil { + t.Fatal(err) + } + if err := r.cancel(run.ID); err != nil { + t.Fatal("repeated stop must be idempotent", err) + } + detail, err := r.getRun(run.ID) + if err != nil || detail.Status != "running" || !detail.Executing || !detail.CancelRequested || detail.FinishedAt != "" || detail.Steps[0].Status != "running" || detail.Steps[0].FinishedAt != "" { + t.Fatalf("stop request falsely settled execution: %+v %v", detail, err) + } + request.RequestKey = "conflict" + if _, err := r.start(request); err == nil { + t.Fatal("stopping workspace was released") + } + // An actual second server must fail before recovery can touch this live row. + other, err := pipeline.Open(filepath.Join(r.store.Directory(), "test.db")) + if err != nil { + t.Fatal(err) + } + defer other.Close() + if _, err := newRunner(ctx, other); err == nil || !strings.Contains(err.Error(), "another server owns") { + t.Fatalf("second server did not fail before recovery: %v", err) + } + detail, _ = r.store.GetRun(ctx, run.ID) + if detail.Status != "running" || detail.Steps[0].Status != "running" { + t.Fatal("second server recovered live execution", detail) + } + stopExecutor() + r.wg.Wait() + detail, err = r.store.GetRun(ctx, run.ID) + if err != nil || detail.Status != "canceled" || detail.Outcome != "" || detail.FinishedAt == "" || detail.Steps[0].Status != "canceled" { + t.Fatalf("late success beat stop: %+v %v", detail, err) + } + r.mu.Lock() + err = r.checkWorkspaceFree(workspace) + r.mu.Unlock() + if err != nil { + t.Fatal("confirmed stop did not release workspace", err) + } +} + +func TestCleanupUnconfirmedReservesWorkspaceAcrossRecovery(t *testing.T) { + ctx := context.Background() + r, store, pipelineID := submitRunner(t, ctx, pipeline.Template([]string{"one"}, true)) + path := filepath.Join(store.Directory(), "test.db") + defer func() { store.Close() }() + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{}, stage.ErrStageCleanup + } + workspace := t.TempDir() + request := protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "cleanup", Workspace: workspace, Input: map[string]any{"prompt": "task"}} + run, err := r.start(request) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + if err := r.cancel(run.ID); err != nil { + t.Fatal(err) + } + detail, err := r.getRun(run.ID) + if err != nil || detail.Status != "running" || !detail.CleanupUnconfirmed || detail.FinishedAt != "" || !detail.Executing { + t.Fatalf("cleanup failure became terminal: %+v %v", detail, err) + } + request.RequestKey = "blocked" + if _, err := r.start(request); err == nil { + t.Fatal("unconfirmed cleanup released workspace") + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + store, err = pipeline.Open(path) + if err != nil { + t.Fatal(err) + } + recovered, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + // No recoverable session pointer exists in this fixture: fail closed rather + // than claiming a restart proved an external process stopped. + if _, err := recovered.start(request); err == nil { + t.Fatal("restart released unconfirmed workspace without stop evidence") + } + detail, _ = recovered.getRun(run.ID) + if !detail.CleanupUnconfirmed || detail.Status != "running" || detail.FinishedAt != "" { + t.Fatal("recovery invented stop confirmation", detail) + } +} + +func TestWorkspaceOverlapIncludesParentsDescendantsAndSymlinks(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + workspace := t.TempDir() + child := filepath.Join(workspace, "child") + if err := os.Mkdir(child, 0700); err != nil { + t.Fatal(err) + } + alias := filepath.Join(t.TempDir(), "alias") + if err := os.Symlink(child, alias); err != nil { + t.Fatal(err) + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{}, nil + } + // Admission resolves aliases; the overlap check must cover both directions. + for _, pair := range [][2]string{{workspace, child}, {child, workspace}, {workspace, alias}} { + r.mu.Lock() + resolved, _ := filepath.EvalSymlinks(pair[0]) + r.active[99] = activeRun{cancel: func() {}, workspace: resolved} + r.mu.Unlock() + _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "overlap", Workspace: pair[1], Input: map[string]any{"prompt": "task"}}) + if err == nil || !strings.Contains(err.Error(), "already active") { + t.Fatal("overlapping workspace admission", pair, err) + } + } + for _, pair := range [][2]string{{"/repo", "/repo-other"}, {"/repo/a", "/repo/b"}} { + if workspacesOverlap(pair[0], pair[1]) { + t.Fatal("sibling/prefix falsely overlaps", pair) + } + } +} diff --git a/internal/server/file_changes_test.go b/internal/server/file_changes_test.go index fe1518a5..02d4dbed 100644 --- a/internal/server/file_changes_test.go +++ b/internal/server/file_changes_test.go @@ -102,6 +102,12 @@ func TestRetainedStageDiffsAcrossVisitsAndFailures(t *testing.T) { if ending == "cancel" { want = "canceled" } + if ending == "cleanup" { + want = "running" + if !detail.CleanupUnconfirmed || detail.FinishedAt != "" { + t.Fatal("unconfirmed cleanup reported as settled", detail) + } + } if detail.Status != want { t.Fatal("diff changed verdict", detail.Status, want) } diff --git a/internal/server/runevents.go b/internal/server/runevents.go index 9f4d2ad5..350064f7 100644 --- a/internal/server/runevents.go +++ b/internal/server/runevents.go @@ -427,6 +427,8 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) { if in.NotifyMode != "" { into.NotifyMode = in.NotifyMode } + into.CancelRequested = into.CancelRequested || in.CancelRequested + into.CleanupUnconfirmed = into.CleanupUnconfirmed || in.CleanupUnconfirmed if in.Workspace != "" { into.Workspace = in.Workspace } @@ -503,19 +505,21 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner continue } frame := protocol.RunEventFrame{ - NotifyMode: summary.NotifyMode, - RunID: summary.ID, - Workspace: summary.Workspace, - PipelineID: summary.PipelineID, - Name: summary.Name, - Title: summary.Title, - Version: summary.Version, - Status: summary.Status, - Outcome: summary.Outcome, - CurrentStage: summary.CurrentStage, - CompletedSteps: summary.CompletedSteps, - CreatedAt: summary.CreatedAt, - FinishedAt: summary.FinishedAt, + NotifyMode: summary.NotifyMode, + RunID: summary.ID, + Workspace: summary.Workspace, + PipelineID: summary.PipelineID, + Name: summary.Name, + Title: summary.Title, + Version: summary.Version, + Status: summary.Status, + Outcome: summary.Outcome, + CurrentStage: summary.CurrentStage, + CompletedSteps: summary.CompletedSteps, + CreatedAt: summary.CreatedAt, + FinishedAt: summary.FinishedAt, + CancelRequested: summary.CancelRequested, + CleanupUnconfirmed: summary.CleanupUnconfirmed, } if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) { // The connection write failed; the subscription is closed and the diff --git a/internal/server/runner.go b/internal/server/runner.go index 72290f61..db64e4f7 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -23,8 +23,8 @@ import ( "git.a-wan.net/alan/pipelines/internal/turn" ) -// activeRun records one in-flight run's cancellation seam and the resolved -// workspace it is changing. A single map is the only active-run state, so the +// activeRun records an in-flight or cleanup-unconfirmed run's cancellation seam +// and resolved workspace reservation. A single map is the active-run state, so the // workspace cannot drift from the cancel function. type activeRun struct { cancel context.CancelFunc @@ -77,6 +77,9 @@ type runner struct { } func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { + if err := s.AcquireRunnerLock(); err != nil { + return nil, err + } sessions, err := s.ActiveSessionOwners(ctx) if err != nil { return nil, err @@ -101,7 +104,17 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { return nil, err } root := filepath.Join(s.Directory(), "artifacts") - r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession} + active := map[int64]activeRun{} + rows, err := s.ListRuns(ctx) + if err != nil { + return nil, err + } + for _, row := range rows { + if row.CleanupUnconfirmed { + active[row.ID] = activeRun{cancel: func() {}, workspace: row.Workspace} + } + } + r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession} r.schedulerWG.Add(1) go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }() return r, nil @@ -331,24 +344,48 @@ func (r *runner) launch(id int64, workspace string, fn func(context.Context)) { go func() { defer r.wg.Done() defer cancel() - defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }() + defer func() { + r.mu.Lock() + defer r.mu.Unlock() + check, done := context.WithTimeout(context.Background(), 10*time.Second) + defer done() + detail, err := r.store.GetRun(check, id) + if err != nil || detail.CleanupUnconfirmed { + return // Fail closed: execution/cleanup has not settled durably. + } + if detail.Status == protocol.RunStatusWaiting && detail.CancelRequested { + if err := r.store.CancelWaiting(check, id); err != nil { + return + } + r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: detail.Workspace, Status: protocol.RunStatusCanceled, CancelRequested: true, CurrentStage: detail.CurrentStage, FinishedAt: time.Now().UTC().Format(time.RFC3339)}, true) + } + if detail.FinishedAt != "" || detail.Status == protocol.RunStatusWaiting { + delete(r.active, id) + } + }() fn(ctx) }() } -// checkWorkspaceFree refuses a run when an active run is already changing the -// same resolved workspace. The workspace is the only scope a run reserves, and -// one active run per workspace keeps this scan finding at most one match, so the -// message is deterministic. Callers must hold r.mu. +// checkWorkspaceFree refuses a run when an active or cleanup-unconfirmed run +// reserves an overlapping resolved workspace. Callers must hold r.mu. func (r *runner) checkWorkspaceFree(workspace string) error { for id, entry := range r.active { - if entry.workspace == workspace { + if workspacesOverlap(entry.workspace, workspace) { return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace) } } return nil } +func workspacesOverlap(a, b string) bool { + within := func(parent, child string) bool { + rel, err := filepath.Rel(parent, child) + return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) + } + return within(a, b) || within(b, a) +} + func (r *runner) getRun(id int64) (*protocol.RunDetail, error) { r.mu.Lock() defer r.mu.Unlock() @@ -675,32 +712,34 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { func (r *runner) cancel(id int64) error { r.mu.Lock() + defer r.mu.Unlock() entry, ok := r.active[id] - r.mu.Unlock() - // Persist the cancellation before asking the runner to stop. A stage that is - // stuck in a context-unaware provider call must not leave the run "running": - // the operator's Stop is a terminal decision, and the live row/step show it - // immediately. FinishStep/FinishRun only update active rows, so the late - // stage return cannot undo this. if err := r.store.Cancel(r.ctx, id); err != nil { return err } if ok { entry.cancel() + } else { + // Only a parked human run is known to have no live executor. + if err := r.store.CancelWaiting(r.ctx, id); err != nil { + return err + } } r.forgetAttention(id) if r.events != nil { if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil { r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary)) r.emitRun(protocol.RunEventFrame{ - RunID: id, - Workspace: detail.Workspace, - Status: detail.Status, - Outcome: detail.Outcome, - CurrentStage: detail.CurrentStage, - CompletedSteps: detail.CompletedSteps, - FinishedAt: detail.FinishedAt, - }, true) + RunID: id, + Workspace: detail.Workspace, + Status: detail.Status, + Outcome: detail.Outcome, + CurrentStage: detail.CurrentStage, + CompletedSteps: detail.CompletedSteps, + FinishedAt: detail.FinishedAt, + CancelRequested: detail.CancelRequested, + CleanupUnconfirmed: detail.CleanupUnconfirmed, + }, !ok) } } return nil @@ -1265,20 +1304,22 @@ func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) { } func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) { + r.mu.Lock() c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil { - // Cancel wins over a late stage result: the store already marked the run - // canceled and FinishRun only updates queued/running rows. Read the - // actual state so this turn emits and notifies canceled, never a failure - // caused by the canceled step's persistence guard. - if detail, getErr := r.store.GetRun(c, id); getErr == nil && detail != nil && detail.Status == protocol.RunStatusCanceled { - status = protocol.RunStatusCanceled - outcome = detail.Outcome - diagnostic = detail.Diagnostic - } else { - log.Printf("finish run %d: %v", id, err) - } + log.Printf("finish run %d: %v", id, err) + } + // The store atomically makes a stop request win over a late success. + detail, getErr := r.store.GetRun(c, id) + if getErr != nil || detail == nil { + r.mu.Unlock() + return + } + status, outcome, diagnostic = detail.Status, detail.Outcome, detail.Diagnostic + r.mu.Unlock() + if detail.FinishedAt == "" { + return // Cleanup is not confirmed: never emit a false terminal event. } // Push the terminal delta before the attention notification, so a slow ntfy // never delays the live update the app folds. @@ -1289,7 +1330,7 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag Outcome: outcome, CurrentStage: currentStage, CompletedSteps: completed, - FinishedAt: time.Now().UTC().Format(time.RFC3339), + FinishedAt: detail.FinishedAt, }, true) r.notifyRun(id, name, status, sequence, currentStage, diagnostic) // The run is terminal: nothing more can notify for it. @@ -1353,12 +1394,27 @@ func turnOutputOnly(output stage.Values) bool { } func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) { + r.mu.Lock() save, cancel := context.WithTimeout(context.Background(), 10*time.Second) + detail, getErr := r.store.GetRun(save, id) + if getErr != nil { + cancel() + r.mu.Unlock() + return "", "", "", false, getErr + } + if detail.CancelRequested { + step.Status, step.Outcome, step.Target = "canceled", "", "" + step.Diagnostic = "Canceled by operator; execution returned" + } persistErr := r.store.FinishStep(save, id, step) cancel() + r.mu.Unlock() if persistErr != nil { return "", "", "", false, persistErr } + if detail.CancelRequested { + return "canceled", "", step.Diagnostic, true, nil + } if step.Status == "completed" { values, copyErr := copyValues(stage.Values(step.Output)) if copyErr != nil { @@ -1696,6 +1752,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na result, err = stage.DefaultRegistry().Run(stageCtx, def, input) } }() + ownershipCtx, ownershipCancel := context.WithTimeout(context.Background(), 10*time.Second) + activeSessions, ownershipErr := r.store.ActiveAgentSessionsForRun(ownershipCtx, id) + ownershipCancel() + if ownershipErr != nil || len(activeSessions) > 0 { + err = errors.Join(stage.ErrStageCleanup, err, ownershipErr) + } + if ctx.Err() != nil && !errors.Is(err, stage.ErrStageCleanup) { + err = ctx.Err() // Ignore a context-unaware executor's late success. + } if stage.EffectiveType(def.Type) == stage.ModelType && err == nil { ensureTurnOutput(visitTurn, result) } @@ -1707,6 +1772,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na if errors.Is(err, stage.ErrStageCleanup) { changes.Status = "partial" changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing." + save, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if markErr := r.store.MarkCleanupUnconfirmed(save, id); markErr != nil { + log.Printf("mark cleanup unconfirmed for run %d: %v", id, markErr) + } + cancel() } // Export the stage's session into its own artifacts directory before the // bounded scan, so the existing listing and serving paths find the files @@ -1790,6 +1860,17 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na step.Transcript = stage.RetainSession(step.Transcript, stageSession) } step.Target = def.Edges[step.Outcome] + if errors.Is(err, stage.ErrStageCleanup) { + // Retain the partial evidence without pretending that the agent stopped. + step.Status, step.Outcome, step.Target = "running", "", "" + save, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if saveErr := r.store.FinishStep(save, id, step); saveErr != nil { + log.Printf("save cleanup evidence for run %d: %v", id, saveErr) + } + cancel() + r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CleanupUnconfirmed: true}, false) + return + } step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads} if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits { for _, outcome := range selected.Limits.On { @@ -1823,6 +1904,10 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na finish("failed", "error", finishErr.Error()) return } + if ctx.Err() != nil { + finish("canceled", "", ctx.Err().Error()) + return + } // Advance the in-memory UI-surface evidence only after the step is // persisted, so the skip guard agrees with what the store holds. state.recordChanges(step.FileChanges, step.Artifacts) @@ -1856,13 +1941,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na Artifacts: len(step.Artifacts), }, }, false) - if useAgent || errors.Is(err, stage.ErrStageCleanup) { - sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id) - if checkErr != nil || len(sessions) > 0 { - finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work") - return - } - } if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return diff --git a/internal/server/runner_test.go b/internal/server/runner_test.go index 31b8921a..ac89f77a 100644 --- a/internal/server/runner_test.go +++ b/internal/server/runner_test.go @@ -135,7 +135,12 @@ func TestHandlerWiresRunTitleGenerator(t *testing.T) { } defer store.Close() configured := HandlerWithOptions(ctx, Options{RunTitles: summarise.GenerateTitle}, store).(*runHandler) - bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler) + bareStore, err := pipeline.Open(filepath.Join(t.TempDir(), "bare.db")) + if err != nil { + t.Fatal(err) + } + defer bareStore.Close() + bare := HandlerWithOptions(ctx, Options{}, bareStore).(*runHandler) defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }() if configured.runner.title == nil { t.Fatal("configured RunTitles was not applied to the runner") @@ -461,7 +466,12 @@ func TestHandlerWiresStepVerdictGenerator(t *testing.T) { } defer store.Close() configured := HandlerWithOptions(ctx, Options{StepVerdicts: summarise.GenerateVerdictReason}, store).(*runHandler) - bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler) + bareStore, err := pipeline.Open(filepath.Join(t.TempDir(), "bare.db")) + if err != nil { + t.Fatal(err) + } + defer bareStore.Close() + bare := HandlerWithOptions(ctx, Options{}, bareStore).(*runHandler) defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }() if configured.runner.verdict == nil { t.Fatal("configured StepVerdicts was not applied to the runner") diff --git a/internal/server/session_export_test.go b/internal/server/session_export_test.go index 4aa25bba..013c5496 100644 --- a/internal/server/session_export_test.go +++ b/internal/server/session_export_test.go @@ -71,7 +71,7 @@ func sessionExportRunner(t *testing.T, backend string, seam func(stage.SessionEx var runID int64 handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { <-ready - if err := handler.runner.store.RunSession(ctx, runID, 1, "ses_run", true, backend, ""); err != nil { + if err := handler.runner.store.RunSession(ctx, runID, 1, "ses_run", false, backend, ""); err != nil { return stage.Result{}, err } return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil diff --git a/web/src/actions.js b/web/src/actions.js index 302a4a3e..69b5bbce 100644 --- a/web/src/actions.js +++ b/web/src/actions.js @@ -154,7 +154,7 @@ export class DecisionDraft { /** Cancel is offered only while the run is executing. */ export function canCancel(run) { - return Boolean(run?.executing); + return Boolean(run?.executing) && !run.cancel_requested && !run.cleanup_unconfirmed; } /** Retry is offered only for a loaded, non-executing, `failed` run. */ diff --git a/web/src/runEvents.js b/web/src/runEvents.js index d4b0925b..e0dc4f38 100644 --- a/web/src/runEvents.js +++ b/web/src/runEvents.js @@ -181,6 +181,8 @@ function runStateFields(current) { created_at: current.created_at ?? "", finished_at: current.finished_at ?? "", executing: Boolean(current.executing), + cancel_requested: Boolean(current.cancel_requested), + cleanup_unconfirmed: Boolean(current.cleanup_unconfirmed), }; } @@ -210,6 +212,8 @@ function applyRunStateFields(current, frame) { created_at: blank(frame.created_at) ? before.created_at : frame.created_at, finished_at: blank(frame.finished_at) ? before.finished_at : frame.finished_at, executing: executingFor(status, before.executing), + cancel_requested: before.cancel_requested || frame.cancel_requested === true, + cleanup_unconfirmed: before.cleanup_unconfirmed || frame.cleanup_unconfirmed === true, }; const changed = merged.notify_mode !== before.notify_mode || @@ -220,7 +224,9 @@ function applyRunStateFields(current, frame) { merged.completed_steps !== before.completed_steps || merged.created_at !== before.created_at || merged.finished_at !== before.finished_at || - merged.executing !== before.executing; + merged.executing !== before.executing || + merged.cancel_requested !== before.cancel_requested || + merged.cleanup_unconfirmed !== before.cleanup_unconfirmed; return changed ? { ...current, ...merged } : current; } @@ -244,6 +250,8 @@ export function newRunSummaryFromFrame(frame) { version: Number(frame.version) || 0, status: frame.status ?? "", executing: executingFor(frame.status, true), + cancel_requested: frame.cancel_requested === true, + cleanup_unconfirmed: frame.cleanup_unconfirmed === true, outcome: frame.outcome ?? "", current_stage: frame.current_stage ?? "", created_at: frame.created_at ?? "", @@ -361,6 +369,8 @@ export function runDetailNeedsFullSync(frame) { return ( stepSettled || stepWaiting || + frame?.cancel_requested === true || + frame?.cleanup_unconfirmed === true || isTerminalRunStatus(frame?.status) || isWaitingRunStatus(frame?.status) ); diff --git a/web/src/runs.js b/web/src/runs.js index 7737f6b5..f87ab919 100644 --- a/web/src/runs.js +++ b/web/src/runs.js @@ -37,6 +37,14 @@ export const RUN_STATES = [ * states so one chip set renders list, detail and timeline). Mirrors * runStateOf: blank or unrecognised input is the safe `unknown` fallback. */ +export function runStatusLabel(run) { + if (["queued", "running", "waiting"].includes(run?.status)) { + if (run.cleanup_unconfirmed) return "Cleanup unconfirmed — workspace reserved"; + if (run.cancel_requested) return "Stopping — waiting for execution to stop"; + } + return run?.status || "unknown"; +} + export function runStateOf(status) { switch (String(status ?? "").trim().toLowerCase()) { case "running": diff --git a/web/src/views/runDetail.js b/web/src/views/runDetail.js index 7c1cf5ce..6c95cfda 100644 --- a/web/src/views/runDetail.js +++ b/web/src/views/runDetail.js @@ -6,7 +6,7 @@ import { h, kvMono, canCopyLink, canShare, copyArtifactLink, shareArtifact } from "../dom.js"; import { isExecuting, changeKindTone, describeError } from "../format.js"; import { resolveArtifactUrl } from "../serverUrl.js"; -import { runStateOf } from "../runs.js"; +import { runStateOf, runStatusLabel } from "../runs.js"; import { PIPELINE_GRAPH_EMPTY, RUN_GRAPH_MISSING, mount as mountGraph } from "../graph.js"; import { writeGraphExpanded } from "../graphPrefs.js"; import { @@ -135,11 +135,11 @@ function renderRunDetail(now = Date.now()) { if (decision) pane.append(renderDecisionCard(decision)); const status = runStateOf(d.status); const kicker = h("div", { class: "run-kicker" }); - kicker.append(h("span", { class: `chip chip-${status}`, text: d.status || status })); + kicker.append(h("span", { class: `chip chip-${status}`, text: runStatusLabel(d) })); if (String(d.outcome ?? "").trim()) { kicker.append(h("span", { class: "muted", text: `outcome ${d.outcome}` })); } - if (d.executing) kicker.append(h("span", { class: "muted", text: "executing" })); + if (d.executing && !d.cleanup_unconfirmed) kicker.append(h("span", { class: "muted", text: "executing" })); pane.append(kicker); pane.append( h("h1", { diff --git a/web/src/views/runs.js b/web/src/views/runs.js index 18e4ba70..eae3510f 100644 --- a/web/src/views/runs.js +++ b/web/src/views/runs.js @@ -9,6 +9,7 @@ import { RUN_TIME_RANGES, RUN_SORTS, runStateOf, + runStatusLabel, runIdentity, versionLabel, progressText, @@ -125,7 +126,10 @@ function renderRunRow(run, now) { }); const head = h("span", { class: "run-row-head" }); - head.append(statusIcon(status, status.charAt(0).toUpperCase() + status.slice(1))); + head.append(statusIcon(status, runStatusLabel(run))); + if (run.cancel_requested || run.cleanup_unconfirmed) { + head.append(h("span", { class: "muted", text: runStatusLabel(run) })); + } if (run.executing) head.append(h("span", { class: "spinner", "aria-hidden": "true" })); // The pipeline is the list's anchor: it names what ran, before the task title. head.append( diff --git a/web/test/executionSafety.test.mjs b/web/test/executionSafety.test.mjs new file mode 100644 index 00000000..0853f509 --- /dev/null +++ b/web/test/executionSafety.test.mjs @@ -0,0 +1,26 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { canCancel } from "../src/actions.js"; +import { runStatusLabel } from "../src/runs.js"; +import { foldRunEvent, foldRunEventDetail } from "../src/runEvents.js"; + +test("stop intent stays nonterminal, disables Stop, and folds into live rows", () => { + const run = { id: 4, status: "running", executing: true, steps: [] }; + const frame = { run_id: 4, status: "running", cancel_requested: true }; + const updated = foldRunEvent([run], frame).runs[0]; + assert.equal(updated.status, "running"); + assert.equal(updated.executing, true); + assert.equal(canCancel(updated), false); + assert.match(runStatusLabel(updated), /Stopping/); + assert.equal(foldRunEventDetail(run, frame).cancel_requested, true); +}); + +test("cleanup uncertainty remains visible until a terminal observation", () => { + const run = { id: 4, status: "running", executing: true }; + const pending = foldRunEvent([run], { run_id: 4, cleanup_unconfirmed: true }).runs[0]; + assert.equal(canCancel(pending), false); + assert.match(runStatusLabel(pending), /Cleanup unconfirmed.*workspace reserved/); + const terminal = foldRunEvent([pending], { run_id: 4, status: "canceled" }).runs[0]; + assert.equal(terminal.executing, false); + assert.equal(runStatusLabel(terminal), "canceled"); +});