fix: keep cancellation pending until execution cleanup is confirmed

This commit is contained in:
2026-10-20 15:47:07 +01:00
parent 1cf5156c77
commit b96b9d94a6
26 changed files with 690 additions and 142 deletions
+3
View File
@@ -42,6 +42,9 @@ func main() {
log.Fatal(err)
}
defer store.Close()
if err := store.AcquireServerLock(); err != nil {
log.Fatal(err)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
artifactsRoot := filepath.Join(filepath.Dir(*db), "artifacts")
+23 -1
View File
@@ -166,13 +166,35 @@ Every submission, including agent-issued `submit`/`rpc`, still uses server valid
## Drafts in the TUI
The server runs one active run per resolved workspace. A new request key in a
The server runs one active run per resolved workspace, including overlapping
parent/descendant directories (and symlink aliases). Separate sibling worktrees
remain independent. A new request key in a
different workspace is admitted immediately and executes concurrently; a second
run in a workspace an active run is already changing is refused, naming the
conflicting run and advising you to wait or use a different workspace (for
example another checkout of the repository). Reusing the original request key
reattaches idempotently.
`cancel ID` / `cancel_run` records `cancel_requested: true` before signaling the
executor. The reply can still have `status: running` (or `queued`) and no
`finished_at`: this means **stopping**, not stopped. The workspace stays reserved
until execution returns and owned-session cleanup is confirmed; late success
cannot override cancellation. A parked human run has no executor and cancels
immediately. Repeating Stop while stopping is safe.
If cleanup cannot be confirmed, the run stays nonterminal with
`cleanup_unconfirmed: true`, retains partial step evidence, and reserves its
workspace. The web client labels this explicitly and hides duplicate Stop.
Successful startup recovery interrupts and confirms recorded sessions before
settling the run; without a recoverable session pointer, uncertainty and the
reservation survive restart rather than inventing stop confirmation. Recovery
does not resume a canceled run. A database-sidecar advisory server lock is held
through runner shutdown, so a second server on that database cannot recover or
interrupt the first one's live runs. Keep the sidecar file in place; its existence
is not ownership (the kernel lock is). This is a local-filesystem lock, not a
distributed lease; hard-link database aliases and filesystems without reliable
advisory locking are unsupported.
Use `:templates` for four simple, manual-only starters: task, review, research,
and plan/implement/review. **Submit template** (`s` or Enter) asks for a
name without opening an editor. Enter saves a retained draft, then a separate
+70
View File
@@ -0,0 +1,70 @@
package pipeline
import (
"context"
"path/filepath"
"testing"
)
func TestCancelRecoveryRequiresCleanupConfirmation(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "recovery.db")
store, err := Open(path)
if err != nil {
t.Fatal(err)
}
defer func() { store.Close() }()
if _, err := store.Submit(ctx, "recovery", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
id, _, err := store.CreateRun(ctx, definition, "recover", t.TempDir(), false, nil, map[string]any{"prompt": "task"})
if err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
t.Fatal(err)
}
if err := store.RunSession(ctx, id, 1, "ses_cleanup", true, "v2", "fixture"); err != nil {
t.Fatal(err)
}
if err := store.Cancel(ctx, id); err != nil {
t.Fatal(err)
}
if err := store.MarkCleanupUnconfirmed(ctx, id); err != nil {
t.Fatal(err)
}
if err := store.FinishRun(ctx, id, "canceled", "", "", nil); err == nil {
t.Fatal("unconfirmed cleanup accepted terminal state")
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = Open(path)
if err != nil {
t.Fatal(err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
detail, _ := store.GetRun(ctx, id)
if detail.Status != "running" || !detail.CancelRequested || !detail.CleanupUnconfirmed || !detail.Steps[0].AgentActive || detail.Steps[0].FinishedAt != "" {
t.Fatal("recovery falsely settled unconfirmed execution", detail)
}
owners, err := store.ActiveSessionOwners(ctx)
if err != nil || len(owners) != 1 {
t.Fatalf("lost recovery owner: %+v %v", owners, err)
}
// The runner calls this only after InterruptAndConfirm succeeds.
if err := store.ConfirmRecoveredSession(ctx, owners[0]); err != nil {
t.Fatal(err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
detail, _ = store.GetRun(ctx, id)
if detail.Status != "canceled" || detail.CleanupUnconfirmed || detail.FinishedAt == "" || detail.Steps[0].Status != "canceled" || detail.Steps[0].AgentActive {
t.Fatal("confirmed recovery failed to settle cancellation", detail)
}
}
+13 -3
View File
@@ -183,7 +183,7 @@ func TestResolveDecisionValidUnknownRepeatAndNotWaiting(t *testing.T) {
}
}
func TestCancelRunningRunMarksRunAndLiveStep(t *testing.T) {
func TestCancelRunningRunRecordsIntentUntilExecutionReturns(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "cancel-running.db"))
if err != nil {
@@ -212,8 +212,18 @@ func TestCancelRunningRunMarksRunAndLiveStep(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if detail.Status != "canceled" || detail.Steps[0].Status != "canceled" {
t.Fatalf("running run not canceled immediately: %+v %+v", detail.RunSummary, detail.Steps)
if detail.Status != "running" || detail.Steps[0].Status != "running" || !detail.CancelRequested || detail.FinishedAt != "" {
t.Fatalf("stop request falsely settled execution: %+v %+v", detail.RunSummary, detail.Steps)
}
if err := store.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "gate", Status: "completed", Outcome: "success"}); err != nil {
t.Fatal(err)
}
if err := store.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
detail, err = store.GetRun(ctx, id)
if err != nil || detail.Status != "canceled" || detail.Steps[0].Status != "canceled" {
t.Fatalf("late success beat cancellation: %+v %v", detail, err)
}
if err := store.Cancel(ctx, id); err == nil {
t.Fatal("canceling an already-canceled run succeeded")
+54
View File
@@ -0,0 +1,54 @@
package pipeline
import (
"fmt"
"os"
"path/filepath"
"golang.org/x/sys/unix"
)
// AcquireRunnerLock must precede recovery: a second server must never interrupt
// the first server's live sessions. The kernel releases the advisory lock on
// process exit; the sidecar is deliberately never unlinked (avoiding inode races).
// The store must stay open until all runner goroutines have stopped.
func (s *Store) AcquireRunnerLock() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.runnerClaimed {
return fmt.Errorf("a runner already owns this store")
}
if err := s.acquireServerLock(); err != nil {
return err
}
s.runnerClaimed = true
return nil
}
// AcquireServerLock lets the executable fail before attaching chat or serving
// requests. newRunner also claims it, protecting embedded handler callers.
func (s *Store) AcquireServerLock() error {
s.mu.Lock()
defer s.mu.Unlock()
return s.acquireServerLock()
}
func (s *Store) acquireServerLock() error {
if s.runnerLock != nil {
return nil
}
path, err := filepath.EvalSymlinks(s.path)
if err != nil {
return fmt.Errorf("resolve server database: %w", err)
}
file, err := os.OpenFile(path+".server.lock", os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return err
}
if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil {
file.Close()
return fmt.Errorf("another server owns database %s: %w", path, err)
}
s.runnerLock = file
return nil
}
+60
View File
@@ -0,0 +1,60 @@
package pipeline
import (
"os"
"os/exec"
"path/filepath"
"testing"
)
func TestRunnerLockCrossProcessAndRelease(t *testing.T) {
if path := os.Getenv("PIPELINES_TEST_LOCK_DB"); path != "" {
store, err := Open(path)
if err != nil {
t.Fatal(err)
}
defer store.Close()
if err := store.AcquireRunnerLock(); err == nil {
t.Fatal("child acquired parent's server lock")
}
return
}
path := filepath.Join(t.TempDir(), "lock.db")
store, err := Open(path)
if err != nil {
t.Fatal(err)
}
defer store.Close()
if err := store.AcquireServerLock(); err != nil {
t.Fatal(err)
}
if err := store.AcquireRunnerLock(); err != nil {
t.Fatal(err)
}
if err := store.AcquireRunnerLock(); err == nil {
t.Fatal("two runners claimed one store")
}
cmd := exec.Command(os.Args[0], "-test.run=^TestRunnerLockCrossProcessAndRelease$")
cmd.Env = append(os.Environ(), "PIPELINES_TEST_LOCK_DB="+path)
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("cross-process lock: %v\n%s", err, output)
}
alias := filepath.Join(t.TempDir(), "alias.db")
if err := os.Symlink(path, alias); err != nil {
t.Fatal(err)
}
other, err := Open(alias)
if err != nil {
t.Fatal(err)
}
defer other.Close()
if err := other.AcquireRunnerLock(); err == nil {
t.Fatal("symlink bypassed lock")
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
if err := other.AcquireRunnerLock(); err != nil {
t.Fatal("lock not released with store", err)
}
}
+2 -2
View File
@@ -21,12 +21,12 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
` + runStamps + `,r.diagnostic,
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
var output, stageModels string
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode); err != nil {
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed); err != nil {
return err
}
if stageModels != "" && stageModels != "{}" {
+28 -33
View File
@@ -44,7 +44,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running')", key, id)); err != nil {
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0", key, id)); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
@@ -86,8 +86,16 @@ func (s *Store) FinishStep(ctx context.Context, id int64, step protocol.RunStep)
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,provider=?,model=?,finished_at=CURRENT_TIMESTAMP
WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, step.Provider, step.Model, id, step.Sequence)); err != nil {
var requested bool
if err := tx.QueryRowContext(ctx, "SELECT cancel_requested FROM pipeline_runs WHERE id=?", id).Scan(&requested); err != nil {
return err
}
if requested && step.Status != "running" {
step.Status, step.Outcome, step.Target = "canceled", "", ""
step.Diagnostic = "Canceled by operator; execution returned"
}
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,provider=?,model=?,finished_at=CASE WHEN ?='running' THEN NULL ELSE CURRENT_TIMESTAMP END
WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, step.Provider, step.Model, step.Status, id, step.Sequence)); err != nil {
return err
}
if step.FileChanges != nil {
@@ -288,7 +296,7 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
return err
}
defer tx.Rollback()
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',finished_at=CURRENT_TIMESTAMP WHERE id=? AND status=?`, id, protocol.RunStatusWaiting)
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,diagnostic='Canceled while waiting for a decision',finished_at=CURRENT_TIMESTAMP WHERE id=? AND status=?`, id, protocol.RunStatusWaiting)
if err != nil {
return err
}
@@ -305,35 +313,11 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
return tx.Commit()
}
// Cancel immediately marks an active run and its live step canceled. The
// runner's context is canceled too, but a harness that ignores cancellation (or
// is stuck awaiting a provider) must not leave the operator looking at a run
// that still says "running" after they pressed Stop.
// Cancel records intent only; it is not evidence that the executor stopped.
func (s *Store) Cancel(ctx context.Context, id int64) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs
SET status='canceled', diagnostic=CASE WHEN diagnostic='' THEN 'Canceled by operator' ELSE diagnostic END, finished_at=CURRENT_TIMESTAMP
WHERE id=? AND status IN ('queued','running','waiting')`, id)
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return err
}
if n != 1 {
return fmt.Errorf("run is not active or waiting")
}
if _, err := tx.ExecContext(ctx, `UPDATE run_steps
SET status='canceled', diagnostic=CASE WHEN diagnostic='' THEN 'Canceled by operator' ELSE diagnostic END, finished_at=CURRENT_TIMESTAMP
WHERE run_id=? AND status IN ('running','waiting')`, id); err != nil {
return err
}
return tx.Commit()
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cancel_requested=1,
diagnostic=CASE WHEN cleanup_unconfirmed=1 THEN diagnostic ELSE 'Stop requested; waiting for execution and cleanup to stop' END
WHERE id=? AND status IN ('queued','running','waiting')`, id))
}
func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagnostic string, output map[string]any) error {
@@ -341,5 +325,16 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno
if err != nil {
return err
}
return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id))
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET
status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END,
outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END,
output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id))
}
func (s *Store) MarkCleanupUnconfirmed(ctx context.Context, id int64) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cleanup_unconfirmed=1,
diagnostic='Agent cleanup was not confirmed; workspace remains reserved until successful restart recovery'
WHERE id=? AND status IN ('queued','running')`, id))
}
+7 -2
View File
@@ -62,6 +62,11 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints (
if err := s.migrateRunTitles(); err != nil {
return err
}
for _, column := range []string{"cancel_requested", "cleanup_unconfirmed"} {
if err := s.addColumnIfMissing("pipeline_runs", column, "INTEGER NOT NULL DEFAULT 0"); err != nil {
return err
}
}
if err := s.addColumnIfMissing("pipeline_runs", "notify_mode", "TEXT NOT NULL DEFAULT 'auto'"); err != nil {
return err
}
@@ -184,10 +189,10 @@ func (s *Store) RecoverRuns(ctx context.Context) error {
defer tx.Rollback()
// No replay: pending visits never began. A cleanup agent that was still
// active is marked interrupted, and its run is interrupted below.
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil {
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)", protocol.StepStatusWaiting); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic='Server stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>?", protocol.RunStatusWaiting); err != nil {
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil {
return err
}
return tx.Commit()
+20 -4
View File
@@ -32,8 +32,18 @@ func (s *Store) RunSession(ctx context.Context, id int64, sequence int, session
// ConfirmRecoveredSession clears the agent-active marker after startup has
// interrupted a previously active session, so it no longer blocks new work.
func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error {
_, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID)
return err
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET cleanup_unconfirmed=0 WHERE id=? AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=? AND agent_active=1)`, owner.RunID, owner.RunID); err != nil {
return err
}
return tx.Commit()
}
type SessionOwner struct {
@@ -83,8 +93,14 @@ FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id LEFT JOIN run_session_own
if activeOnly {
query += " AND s.agent_active=1"
}
query += " ORDER BY s.run_id DESC LIMIT ?"
rows, err := s.db.QueryContext(ctx, query, sessionOwnerScanLimit)
query += " ORDER BY s.run_id DESC"
var args []any
if !activeOnly {
query += " LIMIT ?"
args = append(args, sessionOwnerScanLimit)
}
// Recovery cannot silently drop old but still-active owners at the history cap.
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
+16 -5
View File
@@ -18,10 +18,12 @@ import (
)
type Store struct {
db *sql.DB
dir string
path string
mu sync.Mutex
db *sql.DB
dir string
path string
mu sync.Mutex
runnerLock *os.File
runnerClaimed bool
}
// openDB opens one SQLite connection pool for a DSN. Every open path shares it
@@ -73,7 +75,16 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
return s, nil
}
func (s *Store) Close() error { return s.db.Close() }
func (s *Store) Close() error {
err := s.db.Close()
s.mu.Lock()
defer s.mu.Unlock()
if s.runnerLock != nil {
_ = s.runnerLock.Close()
s.runnerLock = nil
}
return err
}
// OpenReadOnly opens an existing database without running schema DDL or
// changing journal mode, so a live database can be queried read-only while its
+12 -10
View File
@@ -29,16 +29,18 @@ type RunEventFrame struct {
Workspace string `json:"workspace,omitempty"`
// PipelineID, Name, Title and Version identify the run's pipeline and its
// operator-facing title, so a new run's row can be built without a fetch.
PipelineID int64 `json:"pipeline_id,omitempty"`
Name string `json:"name,omitempty"`
Title string `json:"title,omitempty"`
Version int `json:"version,omitempty"`
Status string `json:"status,omitempty"`
Outcome string `json:"outcome,omitempty"`
CurrentStage string `json:"current_stage,omitempty"`
CompletedSteps int `json:"completed_steps,omitempty"`
CreatedAt string `json:"created_at,omitempty"`
FinishedAt string `json:"finished_at,omitempty"`
PipelineID int64 `json:"pipeline_id,omitempty"`
Name string `json:"name,omitempty"`
Title string `json:"title,omitempty"`
Version int `json:"version,omitempty"`
Status string `json:"status,omitempty"`
CancelRequested bool `json:"cancel_requested,omitempty"`
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
Outcome string `json:"outcome,omitempty"`
CurrentStage string `json:"current_stage,omitempty"`
CompletedSteps int `json:"completed_steps,omitempty"`
CreatedAt string `json:"created_at,omitempty"`
FinishedAt string `json:"finished_at,omitempty"`
// Artifacts is the number of files recorded for the referenced step, so a
// client can surface artifact capture without reading the step.
Artifacts int `json:"artifacts,omitempty"`
+15 -13
View File
@@ -126,19 +126,21 @@ type RunSummary struct {
// Title is the operator-facing, bounded single-line description of what the
// run is for, generated from its prompt. It is never empty for a new run;
// historical rows predating the title column may omit it.
Title string `json:"title,omitempty"`
Version int `json:"version"`
ContentHash string `json:"content_hash"`
Status string `json:"status"`
Executing bool `json:"executing"`
Outcome string `json:"outcome"`
CurrentStage string `json:"current_stage"`
Workspace string `json:"workspace,omitempty"`
CreatedAt string `json:"created_at"`
FinishedAt string `json:"finished_at"`
Diagnostic string `json:"diagnostic"`
ResultPreview string `json:"result_preview,omitempty"`
CompletedSteps int `json:"completed_steps"`
Title string `json:"title,omitempty"`
Version int `json:"version"`
ContentHash string `json:"content_hash"`
Status string `json:"status"`
CancelRequested bool `json:"cancel_requested,omitempty"`
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
Executing bool `json:"executing"`
Outcome string `json:"outcome"`
CurrentStage string `json:"current_stage"`
Workspace string `json:"workspace,omitempty"`
CreatedAt string `json:"created_at"`
FinishedAt string `json:"finished_at"`
Diagnostic string `json:"diagnostic"`
ResultPreview string `json:"result_preview,omitempty"`
CompletedSteps int `json:"completed_steps"`
// Model and StageModels are the launch-time overrides this run records so it
// stays reproducible: Model covers every model stage and StageModels overrides
// individual stages by key. They are empty for a run that used the pinned
+4 -5
View File
@@ -514,9 +514,8 @@ func waitEnteredString(t *testing.T, entered <-chan string) {
}
}
// TestOwnActiveSessionStillFailsRun proves a run still fails when one of its OWN
// agent sessions is left active after the stage completes.
func TestOwnActiveSessionStillFailsRun(t *testing.T) {
// Active ownership is not a terminal failure: the agent might still mutate files.
func TestOwnActiveSessionKeepsRunUnsettled(t *testing.T) {
_, r, pipelineID := agentStore(t)
workspace := t.TempDir()
ready := make(chan struct{})
@@ -539,7 +538,7 @@ func TestOwnActiveSessionStillFailsRun(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "agent quiescence not confirmed") {
t.Fatalf("own active session did not fail the run: %+v", detail)
if detail.Status != "running" || !detail.CleanupUnconfirmed || detail.FinishedAt != "" {
t.Fatalf("own active session falsely settled the run: %+v", detail)
}
}
+153
View File
@@ -0,0 +1,153 @@
package server
import (
"context"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestCancelWaitsForContextIgnoringExecutor(t *testing.T) {
ctx, r, pipelineID := concurrentStore(t)
entered, release := make(chan struct{}, 1), make(chan struct{})
var releaseOnce sync.Once
stopExecutor := func() { releaseOnce.Do(func() { close(release) }) }
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
entered <- struct{}{}
<-release // Deliberately ignore the canceled context.
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
defer func() { stopExecutor(); r.wg.Wait() }()
workspace := t.TempDir()
request := protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "blocking", Workspace: workspace, Input: map[string]any{"prompt": "task"}}
run, err := r.start(request)
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
if err := r.cancel(run.ID); err != nil {
t.Fatal(err)
}
if err := r.cancel(run.ID); err != nil {
t.Fatal("repeated stop must be idempotent", err)
}
detail, err := r.getRun(run.ID)
if err != nil || detail.Status != "running" || !detail.Executing || !detail.CancelRequested || detail.FinishedAt != "" || detail.Steps[0].Status != "running" || detail.Steps[0].FinishedAt != "" {
t.Fatalf("stop request falsely settled execution: %+v %v", detail, err)
}
request.RequestKey = "conflict"
if _, err := r.start(request); err == nil {
t.Fatal("stopping workspace was released")
}
// An actual second server must fail before recovery can touch this live row.
other, err := pipeline.Open(filepath.Join(r.store.Directory(), "test.db"))
if err != nil {
t.Fatal(err)
}
defer other.Close()
if _, err := newRunner(ctx, other); err == nil || !strings.Contains(err.Error(), "another server owns") {
t.Fatalf("second server did not fail before recovery: %v", err)
}
detail, _ = r.store.GetRun(ctx, run.ID)
if detail.Status != "running" || detail.Steps[0].Status != "running" {
t.Fatal("second server recovered live execution", detail)
}
stopExecutor()
r.wg.Wait()
detail, err = r.store.GetRun(ctx, run.ID)
if err != nil || detail.Status != "canceled" || detail.Outcome != "" || detail.FinishedAt == "" || detail.Steps[0].Status != "canceled" {
t.Fatalf("late success beat stop: %+v %v", detail, err)
}
r.mu.Lock()
err = r.checkWorkspaceFree(workspace)
r.mu.Unlock()
if err != nil {
t.Fatal("confirmed stop did not release workspace", err)
}
}
func TestCleanupUnconfirmedReservesWorkspaceAcrossRecovery(t *testing.T) {
ctx := context.Background()
r, store, pipelineID := submitRunner(t, ctx, pipeline.Template([]string{"one"}, true))
path := filepath.Join(store.Directory(), "test.db")
defer func() { store.Close() }()
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, stage.ErrStageCleanup
}
workspace := t.TempDir()
request := protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "cleanup", Workspace: workspace, Input: map[string]any{"prompt": "task"}}
run, err := r.start(request)
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if err := r.cancel(run.ID); err != nil {
t.Fatal(err)
}
detail, err := r.getRun(run.ID)
if err != nil || detail.Status != "running" || !detail.CleanupUnconfirmed || detail.FinishedAt != "" || !detail.Executing {
t.Fatalf("cleanup failure became terminal: %+v %v", detail, err)
}
request.RequestKey = "blocked"
if _, err := r.start(request); err == nil {
t.Fatal("unconfirmed cleanup released workspace")
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
recovered, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
// No recoverable session pointer exists in this fixture: fail closed rather
// than claiming a restart proved an external process stopped.
if _, err := recovered.start(request); err == nil {
t.Fatal("restart released unconfirmed workspace without stop evidence")
}
detail, _ = recovered.getRun(run.ID)
if !detail.CleanupUnconfirmed || detail.Status != "running" || detail.FinishedAt != "" {
t.Fatal("recovery invented stop confirmation", detail)
}
}
func TestWorkspaceOverlapIncludesParentsDescendantsAndSymlinks(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
workspace := t.TempDir()
child := filepath.Join(workspace, "child")
if err := os.Mkdir(child, 0700); err != nil {
t.Fatal(err)
}
alias := filepath.Join(t.TempDir(), "alias")
if err := os.Symlink(child, alias); err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, nil
}
// Admission resolves aliases; the overlap check must cover both directions.
for _, pair := range [][2]string{{workspace, child}, {child, workspace}, {workspace, alias}} {
r.mu.Lock()
resolved, _ := filepath.EvalSymlinks(pair[0])
r.active[99] = activeRun{cancel: func() {}, workspace: resolved}
r.mu.Unlock()
_, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "overlap", Workspace: pair[1], Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), "already active") {
t.Fatal("overlapping workspace admission", pair, err)
}
}
for _, pair := range [][2]string{{"/repo", "/repo-other"}, {"/repo/a", "/repo/b"}} {
if workspacesOverlap(pair[0], pair[1]) {
t.Fatal("sibling/prefix falsely overlaps", pair)
}
}
}
+6
View File
@@ -102,6 +102,12 @@ func TestRetainedStageDiffsAcrossVisitsAndFailures(t *testing.T) {
if ending == "cancel" {
want = "canceled"
}
if ending == "cleanup" {
want = "running"
if !detail.CleanupUnconfirmed || detail.FinishedAt != "" {
t.Fatal("unconfirmed cleanup reported as settled", detail)
}
}
if detail.Status != want {
t.Fatal("diff changed verdict", detail.Status, want)
}
+17 -13
View File
@@ -427,6 +427,8 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
if in.NotifyMode != "" {
into.NotifyMode = in.NotifyMode
}
into.CancelRequested = into.CancelRequested || in.CancelRequested
into.CleanupUnconfirmed = into.CleanupUnconfirmed || in.CleanupUnconfirmed
if in.Workspace != "" {
into.Workspace = in.Workspace
}
@@ -503,19 +505,21 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
continue
}
frame := protocol.RunEventFrame{
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
CancelRequested: summary.CancelRequested,
CleanupUnconfirmed: summary.CleanupUnconfirmed,
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
+120 -42
View File
@@ -23,8 +23,8 @@ import (
"git.a-wan.net/alan/pipelines/internal/turn"
)
// activeRun records one in-flight run's cancellation seam and the resolved
// workspace it is changing. A single map is the only active-run state, so the
// activeRun records an in-flight or cleanup-unconfirmed run's cancellation seam
// and resolved workspace reservation. A single map is the active-run state, so the
// workspace cannot drift from the cancel function.
type activeRun struct {
cancel context.CancelFunc
@@ -77,6 +77,9 @@ type runner struct {
}
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
if err := s.AcquireRunnerLock(); err != nil {
return nil, err
}
sessions, err := s.ActiveSessionOwners(ctx)
if err != nil {
return nil, err
@@ -101,7 +104,17 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
return nil, err
}
root := filepath.Join(s.Directory(), "artifacts")
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
active := map[int64]activeRun{}
rows, err := s.ListRuns(ctx)
if err != nil {
return nil, err
}
for _, row := range rows {
if row.CleanupUnconfirmed {
active[row.ID] = activeRun{cancel: func() {}, workspace: row.Workspace}
}
}
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
r.schedulerWG.Add(1)
go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }()
return r, nil
@@ -331,24 +344,48 @@ func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
go func() {
defer r.wg.Done()
defer cancel()
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
defer func() {
r.mu.Lock()
defer r.mu.Unlock()
check, done := context.WithTimeout(context.Background(), 10*time.Second)
defer done()
detail, err := r.store.GetRun(check, id)
if err != nil || detail.CleanupUnconfirmed {
return // Fail closed: execution/cleanup has not settled durably.
}
if detail.Status == protocol.RunStatusWaiting && detail.CancelRequested {
if err := r.store.CancelWaiting(check, id); err != nil {
return
}
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: detail.Workspace, Status: protocol.RunStatusCanceled, CancelRequested: true, CurrentStage: detail.CurrentStage, FinishedAt: time.Now().UTC().Format(time.RFC3339)}, true)
}
if detail.FinishedAt != "" || detail.Status == protocol.RunStatusWaiting {
delete(r.active, id)
}
}()
fn(ctx)
}()
}
// checkWorkspaceFree refuses a run when an active run is already changing the
// same resolved workspace. The workspace is the only scope a run reserves, and
// one active run per workspace keeps this scan finding at most one match, so the
// message is deterministic. Callers must hold r.mu.
// checkWorkspaceFree refuses a run when an active or cleanup-unconfirmed run
// reserves an overlapping resolved workspace. Callers must hold r.mu.
func (r *runner) checkWorkspaceFree(workspace string) error {
for id, entry := range r.active {
if entry.workspace == workspace {
if workspacesOverlap(entry.workspace, workspace) {
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
}
}
return nil
}
func workspacesOverlap(a, b string) bool {
within := func(parent, child string) bool {
rel, err := filepath.Rel(parent, child)
return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator))
}
return within(a, b) || within(b, a)
}
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
r.mu.Lock()
defer r.mu.Unlock()
@@ -675,32 +712,34 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
func (r *runner) cancel(id int64) error {
r.mu.Lock()
defer r.mu.Unlock()
entry, ok := r.active[id]
r.mu.Unlock()
// Persist the cancellation before asking the runner to stop. A stage that is
// stuck in a context-unaware provider call must not leave the run "running":
// the operator's Stop is a terminal decision, and the live row/step show it
// immediately. FinishStep/FinishRun only update active rows, so the late
// stage return cannot undo this.
if err := r.store.Cancel(r.ctx, id); err != nil {
return err
}
if ok {
entry.cancel()
} else {
// Only a parked human run is known to have no live executor.
if err := r.store.CancelWaiting(r.ctx, id); err != nil {
return err
}
}
r.forgetAttention(id)
if r.events != nil {
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil {
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: detail.Workspace,
Status: detail.Status,
Outcome: detail.Outcome,
CurrentStage: detail.CurrentStage,
CompletedSteps: detail.CompletedSteps,
FinishedAt: detail.FinishedAt,
}, true)
RunID: id,
Workspace: detail.Workspace,
Status: detail.Status,
Outcome: detail.Outcome,
CurrentStage: detail.CurrentStage,
CompletedSteps: detail.CompletedSteps,
FinishedAt: detail.FinishedAt,
CancelRequested: detail.CancelRequested,
CleanupUnconfirmed: detail.CleanupUnconfirmed,
}, !ok)
}
}
return nil
@@ -1265,20 +1304,22 @@ func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
}
func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) {
r.mu.Lock()
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
// Cancel wins over a late stage result: the store already marked the run
// canceled and FinishRun only updates queued/running rows. Read the
// actual state so this turn emits and notifies canceled, never a failure
// caused by the canceled step's persistence guard.
if detail, getErr := r.store.GetRun(c, id); getErr == nil && detail != nil && detail.Status == protocol.RunStatusCanceled {
status = protocol.RunStatusCanceled
outcome = detail.Outcome
diagnostic = detail.Diagnostic
} else {
log.Printf("finish run %d: %v", id, err)
}
log.Printf("finish run %d: %v", id, err)
}
// The store atomically makes a stop request win over a late success.
detail, getErr := r.store.GetRun(c, id)
if getErr != nil || detail == nil {
r.mu.Unlock()
return
}
status, outcome, diagnostic = detail.Status, detail.Outcome, detail.Diagnostic
r.mu.Unlock()
if detail.FinishedAt == "" {
return // Cleanup is not confirmed: never emit a false terminal event.
}
// Push the terminal delta before the attention notification, so a slow ntfy
// never delays the live update the app folds.
@@ -1289,7 +1330,7 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
Outcome: outcome,
CurrentStage: currentStage,
CompletedSteps: completed,
FinishedAt: time.Now().UTC().Format(time.RFC3339),
FinishedAt: detail.FinishedAt,
}, true)
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
// The run is terminal: nothing more can notify for it.
@@ -1353,12 +1394,27 @@ func turnOutputOnly(output stage.Values) bool {
}
func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) {
r.mu.Lock()
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
detail, getErr := r.store.GetRun(save, id)
if getErr != nil {
cancel()
r.mu.Unlock()
return "", "", "", false, getErr
}
if detail.CancelRequested {
step.Status, step.Outcome, step.Target = "canceled", "", ""
step.Diagnostic = "Canceled by operator; execution returned"
}
persistErr := r.store.FinishStep(save, id, step)
cancel()
r.mu.Unlock()
if persistErr != nil {
return "", "", "", false, persistErr
}
if detail.CancelRequested {
return "canceled", "", step.Diagnostic, true, nil
}
if step.Status == "completed" {
values, copyErr := copyValues(stage.Values(step.Output))
if copyErr != nil {
@@ -1696,6 +1752,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
result, err = stage.DefaultRegistry().Run(stageCtx, def, input)
}
}()
ownershipCtx, ownershipCancel := context.WithTimeout(context.Background(), 10*time.Second)
activeSessions, ownershipErr := r.store.ActiveAgentSessionsForRun(ownershipCtx, id)
ownershipCancel()
if ownershipErr != nil || len(activeSessions) > 0 {
err = errors.Join(stage.ErrStageCleanup, err, ownershipErr)
}
if ctx.Err() != nil && !errors.Is(err, stage.ErrStageCleanup) {
err = ctx.Err() // Ignore a context-unaware executor's late success.
}
if stage.EffectiveType(def.Type) == stage.ModelType && err == nil {
ensureTurnOutput(visitTurn, result)
}
@@ -1707,6 +1772,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
if errors.Is(err, stage.ErrStageCleanup) {
changes.Status = "partial"
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
if markErr := r.store.MarkCleanupUnconfirmed(save, id); markErr != nil {
log.Printf("mark cleanup unconfirmed for run %d: %v", id, markErr)
}
cancel()
}
// Export the stage's session into its own artifacts directory before the
// bounded scan, so the existing listing and serving paths find the files
@@ -1790,6 +1860,17 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
}
step.Target = def.Edges[step.Outcome]
if errors.Is(err, stage.ErrStageCleanup) {
// Retain the partial evidence without pretending that the agent stopped.
step.Status, step.Outcome, step.Target = "running", "", ""
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
if saveErr := r.store.FinishStep(save, id, step); saveErr != nil {
log.Printf("save cleanup evidence for run %d: %v", id, saveErr)
}
cancel()
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CleanupUnconfirmed: true}, false)
return
}
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
for _, outcome := range selected.Limits.On {
@@ -1823,6 +1904,10 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
finish("failed", "error", finishErr.Error())
return
}
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
// Advance the in-memory UI-surface evidence only after the step is
// persisted, so the skip guard agrees with what the store holds.
state.recordChanges(step.FileChanges, step.Artifacts)
@@ -1856,13 +1941,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
Artifacts: len(step.Artifacts),
},
}, false)
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id)
if checkErr != nil || len(sessions) > 0 {
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
return
}
}
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
+12 -2
View File
@@ -135,7 +135,12 @@ func TestHandlerWiresRunTitleGenerator(t *testing.T) {
}
defer store.Close()
configured := HandlerWithOptions(ctx, Options{RunTitles: summarise.GenerateTitle}, store).(*runHandler)
bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler)
bareStore, err := pipeline.Open(filepath.Join(t.TempDir(), "bare.db"))
if err != nil {
t.Fatal(err)
}
defer bareStore.Close()
bare := HandlerWithOptions(ctx, Options{}, bareStore).(*runHandler)
defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }()
if configured.runner.title == nil {
t.Fatal("configured RunTitles was not applied to the runner")
@@ -461,7 +466,12 @@ func TestHandlerWiresStepVerdictGenerator(t *testing.T) {
}
defer store.Close()
configured := HandlerWithOptions(ctx, Options{StepVerdicts: summarise.GenerateVerdictReason}, store).(*runHandler)
bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler)
bareStore, err := pipeline.Open(filepath.Join(t.TempDir(), "bare.db"))
if err != nil {
t.Fatal(err)
}
defer bareStore.Close()
bare := HandlerWithOptions(ctx, Options{}, bareStore).(*runHandler)
defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }()
if configured.runner.verdict == nil {
t.Fatal("configured StepVerdicts was not applied to the runner")
+1 -1
View File
@@ -71,7 +71,7 @@ func sessionExportRunner(t *testing.T, backend string, seam func(stage.SessionEx
var runID int64
handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
<-ready
if err := handler.runner.store.RunSession(ctx, runID, 1, "ses_run", true, backend, ""); err != nil {
if err := handler.runner.store.RunSession(ctx, runID, 1, "ses_run", false, backend, ""); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
+1 -1
View File
@@ -154,7 +154,7 @@ export class DecisionDraft {
/** Cancel is offered only while the run is executing. */
export function canCancel(run) {
return Boolean(run?.executing);
return Boolean(run?.executing) && !run.cancel_requested && !run.cleanup_unconfirmed;
}
/** Retry is offered only for a loaded, non-executing, `failed` run. */
+11 -1
View File
@@ -181,6 +181,8 @@ function runStateFields(current) {
created_at: current.created_at ?? "",
finished_at: current.finished_at ?? "",
executing: Boolean(current.executing),
cancel_requested: Boolean(current.cancel_requested),
cleanup_unconfirmed: Boolean(current.cleanup_unconfirmed),
};
}
@@ -210,6 +212,8 @@ function applyRunStateFields(current, frame) {
created_at: blank(frame.created_at) ? before.created_at : frame.created_at,
finished_at: blank(frame.finished_at) ? before.finished_at : frame.finished_at,
executing: executingFor(status, before.executing),
cancel_requested: before.cancel_requested || frame.cancel_requested === true,
cleanup_unconfirmed: before.cleanup_unconfirmed || frame.cleanup_unconfirmed === true,
};
const changed =
merged.notify_mode !== before.notify_mode ||
@@ -220,7 +224,9 @@ function applyRunStateFields(current, frame) {
merged.completed_steps !== before.completed_steps ||
merged.created_at !== before.created_at ||
merged.finished_at !== before.finished_at ||
merged.executing !== before.executing;
merged.executing !== before.executing ||
merged.cancel_requested !== before.cancel_requested ||
merged.cleanup_unconfirmed !== before.cleanup_unconfirmed;
return changed ? { ...current, ...merged } : current;
}
@@ -244,6 +250,8 @@ export function newRunSummaryFromFrame(frame) {
version: Number(frame.version) || 0,
status: frame.status ?? "",
executing: executingFor(frame.status, true),
cancel_requested: frame.cancel_requested === true,
cleanup_unconfirmed: frame.cleanup_unconfirmed === true,
outcome: frame.outcome ?? "",
current_stage: frame.current_stage ?? "",
created_at: frame.created_at ?? "",
@@ -361,6 +369,8 @@ export function runDetailNeedsFullSync(frame) {
return (
stepSettled ||
stepWaiting ||
frame?.cancel_requested === true ||
frame?.cleanup_unconfirmed === true ||
isTerminalRunStatus(frame?.status) ||
isWaitingRunStatus(frame?.status)
);
+8
View File
@@ -37,6 +37,14 @@ export const RUN_STATES = [
* states so one chip set renders list, detail and timeline). Mirrors
* runStateOf: blank or unrecognised input is the safe `unknown` fallback.
*/
export function runStatusLabel(run) {
if (["queued", "running", "waiting"].includes(run?.status)) {
if (run.cleanup_unconfirmed) return "Cleanup unconfirmed — workspace reserved";
if (run.cancel_requested) return "Stopping — waiting for execution to stop";
}
return run?.status || "unknown";
}
export function runStateOf(status) {
switch (String(status ?? "").trim().toLowerCase()) {
case "running":
+3 -3
View File
@@ -6,7 +6,7 @@
import { h, kvMono, canCopyLink, canShare, copyArtifactLink, shareArtifact } from "../dom.js";
import { isExecuting, changeKindTone, describeError } from "../format.js";
import { resolveArtifactUrl } from "../serverUrl.js";
import { runStateOf } from "../runs.js";
import { runStateOf, runStatusLabel } from "../runs.js";
import { PIPELINE_GRAPH_EMPTY, RUN_GRAPH_MISSING, mount as mountGraph } from "../graph.js";
import { writeGraphExpanded } from "../graphPrefs.js";
import {
@@ -135,11 +135,11 @@ function renderRunDetail(now = Date.now()) {
if (decision) pane.append(renderDecisionCard(decision));
const status = runStateOf(d.status);
const kicker = h("div", { class: "run-kicker" });
kicker.append(h("span", { class: `chip chip-${status}`, text: d.status || status }));
kicker.append(h("span", { class: `chip chip-${status}`, text: runStatusLabel(d) }));
if (String(d.outcome ?? "").trim()) {
kicker.append(h("span", { class: "muted", text: `outcome ${d.outcome}` }));
}
if (d.executing) kicker.append(h("span", { class: "muted", text: "executing" }));
if (d.executing && !d.cleanup_unconfirmed) kicker.append(h("span", { class: "muted", text: "executing" }));
pane.append(kicker);
pane.append(
h("h1", {
+5 -1
View File
@@ -9,6 +9,7 @@ import {
RUN_TIME_RANGES,
RUN_SORTS,
runStateOf,
runStatusLabel,
runIdentity,
versionLabel,
progressText,
@@ -125,7 +126,10 @@ function renderRunRow(run, now) {
});
const head = h("span", { class: "run-row-head" });
head.append(statusIcon(status, status.charAt(0).toUpperCase() + status.slice(1)));
head.append(statusIcon(status, runStatusLabel(run)));
if (run.cancel_requested || run.cleanup_unconfirmed) {
head.append(h("span", { class: "muted", text: runStatusLabel(run) }));
}
if (run.executing) head.append(h("span", { class: "spinner", "aria-hidden": "true" }));
// The pipeline is the list's anchor: it names what ran, before the task title.
head.append(
+26
View File
@@ -0,0 +1,26 @@
import test from "node:test";
import assert from "node:assert/strict";
import { canCancel } from "../src/actions.js";
import { runStatusLabel } from "../src/runs.js";
import { foldRunEvent, foldRunEventDetail } from "../src/runEvents.js";
test("stop intent stays nonterminal, disables Stop, and folds into live rows", () => {
const run = { id: 4, status: "running", executing: true, steps: [] };
const frame = { run_id: 4, status: "running", cancel_requested: true };
const updated = foldRunEvent([run], frame).runs[0];
assert.equal(updated.status, "running");
assert.equal(updated.executing, true);
assert.equal(canCancel(updated), false);
assert.match(runStatusLabel(updated), /Stopping/);
assert.equal(foldRunEventDetail(run, frame).cancel_requested, true);
});
test("cleanup uncertainty remains visible until a terminal observation", () => {
const run = { id: 4, status: "running", executing: true };
const pending = foldRunEvent([run], { run_id: 4, cleanup_unconfirmed: true }).runs[0];
assert.equal(canCancel(pending), false);
assert.match(runStatusLabel(pending), /Cleanup unconfirmed.*workspace reserved/);
const terminal = foldRunEvent([pending], { run_id: 4, status: "canceled" }).runs[0];
assert.equal(terminal.executing, false);
assert.equal(runStatusLabel(terminal), "canceled");
});