fix: keep cancellation pending until execution cleanup is confirmed
This commit is contained in:
@@ -42,6 +42,9 @@ func main() {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if err := store.AcquireServerLock(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
artifactsRoot := filepath.Join(filepath.Dir(*db), "artifacts")
|
||||
|
||||
+23
-1
@@ -166,13 +166,35 @@ Every submission, including agent-issued `submit`/`rpc`, still uses server valid
|
||||
|
||||
## Drafts in the TUI
|
||||
|
||||
The server runs one active run per resolved workspace. A new request key in a
|
||||
The server runs one active run per resolved workspace, including overlapping
|
||||
parent/descendant directories (and symlink aliases). Separate sibling worktrees
|
||||
remain independent. A new request key in a
|
||||
different workspace is admitted immediately and executes concurrently; a second
|
||||
run in a workspace an active run is already changing is refused, naming the
|
||||
conflicting run and advising you to wait or use a different workspace (for
|
||||
example another checkout of the repository). Reusing the original request key
|
||||
reattaches idempotently.
|
||||
|
||||
`cancel ID` / `cancel_run` records `cancel_requested: true` before signaling the
|
||||
executor. The reply can still have `status: running` (or `queued`) and no
|
||||
`finished_at`: this means **stopping**, not stopped. The workspace stays reserved
|
||||
until execution returns and owned-session cleanup is confirmed; late success
|
||||
cannot override cancellation. A parked human run has no executor and cancels
|
||||
immediately. Repeating Stop while stopping is safe.
|
||||
|
||||
If cleanup cannot be confirmed, the run stays nonterminal with
|
||||
`cleanup_unconfirmed: true`, retains partial step evidence, and reserves its
|
||||
workspace. The web client labels this explicitly and hides duplicate Stop.
|
||||
Successful startup recovery interrupts and confirms recorded sessions before
|
||||
settling the run; without a recoverable session pointer, uncertainty and the
|
||||
reservation survive restart rather than inventing stop confirmation. Recovery
|
||||
does not resume a canceled run. A database-sidecar advisory server lock is held
|
||||
through runner shutdown, so a second server on that database cannot recover or
|
||||
interrupt the first one's live runs. Keep the sidecar file in place; its existence
|
||||
is not ownership (the kernel lock is). This is a local-filesystem lock, not a
|
||||
distributed lease; hard-link database aliases and filesystems without reliable
|
||||
advisory locking are unsupported.
|
||||
|
||||
Use `:templates` for four simple, manual-only starters: task, review, research,
|
||||
and plan/implement/review. **Submit template** (`s` or Enter) asks for a
|
||||
name without opening an editor. Enter saves a retained draft, then a separate
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCancelRecoveryRequiresCleanupConfirmation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "recovery.db")
|
||||
store, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { store.Close() }()
|
||||
if _, err := store.Submit(ctx, "recovery", Template([]string{"one"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(ctx)
|
||||
definition, _ := store.Get(ctx, items[0].ID, 1)
|
||||
id, _, err := store.CreateRun(ctx, definition, "recover", t.TempDir(), false, nil, map[string]any{"prompt": "task"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.RunSession(ctx, id, 1, "ses_cleanup", true, "v2", "fixture"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Cancel(ctx, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.MarkCleanupUnconfirmed(ctx, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishRun(ctx, id, "canceled", "", "", nil); err == nil {
|
||||
t.Fatal("unconfirmed cleanup accepted terminal state")
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, _ := store.GetRun(ctx, id)
|
||||
if detail.Status != "running" || !detail.CancelRequested || !detail.CleanupUnconfirmed || !detail.Steps[0].AgentActive || detail.Steps[0].FinishedAt != "" {
|
||||
t.Fatal("recovery falsely settled unconfirmed execution", detail)
|
||||
}
|
||||
owners, err := store.ActiveSessionOwners(ctx)
|
||||
if err != nil || len(owners) != 1 {
|
||||
t.Fatalf("lost recovery owner: %+v %v", owners, err)
|
||||
}
|
||||
// The runner calls this only after InterruptAndConfirm succeeds.
|
||||
if err := store.ConfirmRecoveredSession(ctx, owners[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, _ = store.GetRun(ctx, id)
|
||||
if detail.Status != "canceled" || detail.CleanupUnconfirmed || detail.FinishedAt == "" || detail.Steps[0].Status != "canceled" || detail.Steps[0].AgentActive {
|
||||
t.Fatal("confirmed recovery failed to settle cancellation", detail)
|
||||
}
|
||||
}
|
||||
@@ -183,7 +183,7 @@ func TestResolveDecisionValidUnknownRepeatAndNotWaiting(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelRunningRunMarksRunAndLiveStep(t *testing.T) {
|
||||
func TestCancelRunningRunRecordsIntentUntilExecutionReturns(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store, err := Open(filepath.Join(t.TempDir(), "cancel-running.db"))
|
||||
if err != nil {
|
||||
@@ -212,8 +212,18 @@ func TestCancelRunningRunMarksRunAndLiveStep(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "canceled" || detail.Steps[0].Status != "canceled" {
|
||||
t.Fatalf("running run not canceled immediately: %+v %+v", detail.RunSummary, detail.Steps)
|
||||
if detail.Status != "running" || detail.Steps[0].Status != "running" || !detail.CancelRequested || detail.FinishedAt != "" {
|
||||
t.Fatalf("stop request falsely settled execution: %+v %+v", detail.RunSummary, detail.Steps)
|
||||
}
|
||||
if err := store.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "gate", Status: "completed", Outcome: "success"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err = store.GetRun(ctx, id)
|
||||
if err != nil || detail.Status != "canceled" || detail.Steps[0].Status != "canceled" {
|
||||
t.Fatalf("late success beat cancellation: %+v %v", detail, err)
|
||||
}
|
||||
if err := store.Cancel(ctx, id); err == nil {
|
||||
t.Fatal("canceling an already-canceled run succeeded")
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// AcquireRunnerLock must precede recovery: a second server must never interrupt
|
||||
// the first server's live sessions. The kernel releases the advisory lock on
|
||||
// process exit; the sidecar is deliberately never unlinked (avoiding inode races).
|
||||
// The store must stay open until all runner goroutines have stopped.
|
||||
func (s *Store) AcquireRunnerLock() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.runnerClaimed {
|
||||
return fmt.Errorf("a runner already owns this store")
|
||||
}
|
||||
if err := s.acquireServerLock(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.runnerClaimed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// AcquireServerLock lets the executable fail before attaching chat or serving
|
||||
// requests. newRunner also claims it, protecting embedded handler callers.
|
||||
func (s *Store) AcquireServerLock() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.acquireServerLock()
|
||||
}
|
||||
|
||||
func (s *Store) acquireServerLock() error {
|
||||
if s.runnerLock != nil {
|
||||
return nil
|
||||
}
|
||||
path, err := filepath.EvalSymlinks(s.path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve server database: %w", err)
|
||||
}
|
||||
file, err := os.OpenFile(path+".server.lock", os.O_CREATE|os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil {
|
||||
file.Close()
|
||||
return fmt.Errorf("another server owns database %s: %w", path, err)
|
||||
}
|
||||
s.runnerLock = file
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunnerLockCrossProcessAndRelease(t *testing.T) {
|
||||
if path := os.Getenv("PIPELINES_TEST_LOCK_DB"); path != "" {
|
||||
store, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if err := store.AcquireRunnerLock(); err == nil {
|
||||
t.Fatal("child acquired parent's server lock")
|
||||
}
|
||||
return
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "lock.db")
|
||||
store, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if err := store.AcquireServerLock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.AcquireRunnerLock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.AcquireRunnerLock(); err == nil {
|
||||
t.Fatal("two runners claimed one store")
|
||||
}
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestRunnerLockCrossProcessAndRelease$")
|
||||
cmd.Env = append(os.Environ(), "PIPELINES_TEST_LOCK_DB="+path)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("cross-process lock: %v\n%s", err, output)
|
||||
}
|
||||
alias := filepath.Join(t.TempDir(), "alias.db")
|
||||
if err := os.Symlink(path, alias); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, err := Open(alias)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer other.Close()
|
||||
if err := other.AcquireRunnerLock(); err == nil {
|
||||
t.Fatal("symlink bypassed lock")
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := other.AcquireRunnerLock(); err != nil {
|
||||
t.Fatal("lock not released with store", err)
|
||||
}
|
||||
}
|
||||
@@ -21,12 +21,12 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime
|
||||
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
|
||||
` + runStamps + `,r.diagnostic,
|
||||
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
|
||||
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode
|
||||
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed
|
||||
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
|
||||
|
||||
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
|
||||
var output, stageModels string
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode); err != nil {
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed); err != nil {
|
||||
return err
|
||||
}
|
||||
if stageModels != "" && stageModels != "{}" {
|
||||
|
||||
@@ -44,7 +44,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running')", key, id)); err != nil {
|
||||
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0", key, id)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
|
||||
@@ -86,8 +86,16 @@ func (s *Store) FinishStep(ctx context.Context, id int64, step protocol.RunStep)
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,provider=?,model=?,finished_at=CURRENT_TIMESTAMP
|
||||
WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, step.Provider, step.Model, id, step.Sequence)); err != nil {
|
||||
var requested bool
|
||||
if err := tx.QueryRowContext(ctx, "SELECT cancel_requested FROM pipeline_runs WHERE id=?", id).Scan(&requested); err != nil {
|
||||
return err
|
||||
}
|
||||
if requested && step.Status != "running" {
|
||||
step.Status, step.Outcome, step.Target = "canceled", "", ""
|
||||
step.Diagnostic = "Canceled by operator; execution returned"
|
||||
}
|
||||
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,provider=?,model=?,finished_at=CASE WHEN ?='running' THEN NULL ELSE CURRENT_TIMESTAMP END
|
||||
WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, step.Provider, step.Model, step.Status, id, step.Sequence)); err != nil {
|
||||
return err
|
||||
}
|
||||
if step.FileChanges != nil {
|
||||
@@ -288,7 +296,7 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',finished_at=CURRENT_TIMESTAMP WHERE id=? AND status=?`, id, protocol.RunStatusWaiting)
|
||||
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,diagnostic='Canceled while waiting for a decision',finished_at=CURRENT_TIMESTAMP WHERE id=? AND status=?`, id, protocol.RunStatusWaiting)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -305,35 +313,11 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// Cancel immediately marks an active run and its live step canceled. The
|
||||
// runner's context is canceled too, but a harness that ignores cancellation (or
|
||||
// is stuck awaiting a provider) must not leave the operator looking at a run
|
||||
// that still says "running" after they pressed Stop.
|
||||
// Cancel records intent only; it is not evidence that the executor stopped.
|
||||
func (s *Store) Cancel(ctx context.Context, id int64) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs
|
||||
SET status='canceled', diagnostic=CASE WHEN diagnostic='' THEN 'Canceled by operator' ELSE diagnostic END, finished_at=CURRENT_TIMESTAMP
|
||||
WHERE id=? AND status IN ('queued','running','waiting')`, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != 1 {
|
||||
return fmt.Errorf("run is not active or waiting")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE run_steps
|
||||
SET status='canceled', diagnostic=CASE WHEN diagnostic='' THEN 'Canceled by operator' ELSE diagnostic END, finished_at=CURRENT_TIMESTAMP
|
||||
WHERE run_id=? AND status IN ('running','waiting')`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cancel_requested=1,
|
||||
diagnostic=CASE WHEN cleanup_unconfirmed=1 THEN diagnostic ELSE 'Stop requested; waiting for execution and cleanup to stop' END
|
||||
WHERE id=? AND status IN ('queued','running','waiting')`, id))
|
||||
}
|
||||
|
||||
func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagnostic string, output map[string]any) error {
|
||||
@@ -341,5 +325,16 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id))
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET
|
||||
status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END,
|
||||
outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
|
||||
diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END,
|
||||
output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id))
|
||||
}
|
||||
|
||||
func (s *Store) MarkCleanupUnconfirmed(ctx context.Context, id int64) error {
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cleanup_unconfirmed=1,
|
||||
diagnostic='Agent cleanup was not confirmed; workspace remains reserved until successful restart recovery'
|
||||
WHERE id=? AND status IN ('queued','running')`, id))
|
||||
}
|
||||
|
||||
@@ -62,6 +62,11 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints (
|
||||
if err := s.migrateRunTitles(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, column := range []string{"cancel_requested", "cleanup_unconfirmed"} {
|
||||
if err := s.addColumnIfMissing("pipeline_runs", column, "INTEGER NOT NULL DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := s.addColumnIfMissing("pipeline_runs", "notify_mode", "TEXT NOT NULL DEFAULT 'auto'"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -184,10 +189,10 @@ func (s *Store) RecoverRuns(ctx context.Context) error {
|
||||
defer tx.Rollback()
|
||||
// No replay: pending visits never began. A cleanup agent that was still
|
||||
// active is marked interrupted, and its run is interrupted below.
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)", protocol.StepStatusWaiting); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic='Server stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>?", protocol.RunStatusWaiting); err != nil {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
|
||||
@@ -32,8 +32,18 @@ func (s *Store) RunSession(ctx context.Context, id int64, sequence int, session
|
||||
// ConfirmRecoveredSession clears the agent-active marker after startup has
|
||||
// interrupted a previously active session, so it no longer blocks new work.
|
||||
func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error {
|
||||
_, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID)
|
||||
return err
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET cleanup_unconfirmed=0 WHERE id=? AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=? AND agent_active=1)`, owner.RunID, owner.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
type SessionOwner struct {
|
||||
@@ -83,8 +93,14 @@ FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id LEFT JOIN run_session_own
|
||||
if activeOnly {
|
||||
query += " AND s.agent_active=1"
|
||||
}
|
||||
query += " ORDER BY s.run_id DESC LIMIT ?"
|
||||
rows, err := s.db.QueryContext(ctx, query, sessionOwnerScanLimit)
|
||||
query += " ORDER BY s.run_id DESC"
|
||||
var args []any
|
||||
if !activeOnly {
|
||||
query += " LIMIT ?"
|
||||
args = append(args, sessionOwnerScanLimit)
|
||||
}
|
||||
// Recovery cannot silently drop old but still-active owners at the history cap.
|
||||
rows, err := s.db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -18,10 +18,12 @@ import (
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
dir string
|
||||
path string
|
||||
mu sync.Mutex
|
||||
db *sql.DB
|
||||
dir string
|
||||
path string
|
||||
mu sync.Mutex
|
||||
runnerLock *os.File
|
||||
runnerClaimed bool
|
||||
}
|
||||
|
||||
// openDB opens one SQLite connection pool for a DSN. Every open path shares it
|
||||
@@ -73,7 +75,16 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
func (s *Store) Close() error {
|
||||
err := s.db.Close()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.runnerLock != nil {
|
||||
_ = s.runnerLock.Close()
|
||||
s.runnerLock = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// OpenReadOnly opens an existing database without running schema DDL or
|
||||
// changing journal mode, so a live database can be queried read-only while its
|
||||
|
||||
@@ -29,16 +29,18 @@ type RunEventFrame struct {
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
// PipelineID, Name, Title and Version identify the run's pipeline and its
|
||||
// operator-facing title, so a new run's row can be built without a fetch.
|
||||
PipelineID int64 `json:"pipeline_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Version int `json:"version,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
CurrentStage string `json:"current_stage,omitempty"`
|
||||
CompletedSteps int `json:"completed_steps,omitempty"`
|
||||
CreatedAt string `json:"created_at,omitempty"`
|
||||
FinishedAt string `json:"finished_at,omitempty"`
|
||||
PipelineID int64 `json:"pipeline_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Version int `json:"version,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
CancelRequested bool `json:"cancel_requested,omitempty"`
|
||||
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
CurrentStage string `json:"current_stage,omitempty"`
|
||||
CompletedSteps int `json:"completed_steps,omitempty"`
|
||||
CreatedAt string `json:"created_at,omitempty"`
|
||||
FinishedAt string `json:"finished_at,omitempty"`
|
||||
// Artifacts is the number of files recorded for the referenced step, so a
|
||||
// client can surface artifact capture without reading the step.
|
||||
Artifacts int `json:"artifacts,omitempty"`
|
||||
|
||||
+15
-13
@@ -126,19 +126,21 @@ type RunSummary struct {
|
||||
// Title is the operator-facing, bounded single-line description of what the
|
||||
// run is for, generated from its prompt. It is never empty for a new run;
|
||||
// historical rows predating the title column may omit it.
|
||||
Title string `json:"title,omitempty"`
|
||||
Version int `json:"version"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
Status string `json:"status"`
|
||||
Executing bool `json:"executing"`
|
||||
Outcome string `json:"outcome"`
|
||||
CurrentStage string `json:"current_stage"`
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
Diagnostic string `json:"diagnostic"`
|
||||
ResultPreview string `json:"result_preview,omitempty"`
|
||||
CompletedSteps int `json:"completed_steps"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Version int `json:"version"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
Status string `json:"status"`
|
||||
CancelRequested bool `json:"cancel_requested,omitempty"`
|
||||
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
|
||||
Executing bool `json:"executing"`
|
||||
Outcome string `json:"outcome"`
|
||||
CurrentStage string `json:"current_stage"`
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
Diagnostic string `json:"diagnostic"`
|
||||
ResultPreview string `json:"result_preview,omitempty"`
|
||||
CompletedSteps int `json:"completed_steps"`
|
||||
// Model and StageModels are the launch-time overrides this run records so it
|
||||
// stays reproducible: Model covers every model stage and StageModels overrides
|
||||
// individual stages by key. They are empty for a run that used the pinned
|
||||
|
||||
@@ -514,9 +514,8 @@ func waitEnteredString(t *testing.T, entered <-chan string) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestOwnActiveSessionStillFailsRun proves a run still fails when one of its OWN
|
||||
// agent sessions is left active after the stage completes.
|
||||
func TestOwnActiveSessionStillFailsRun(t *testing.T) {
|
||||
// Active ownership is not a terminal failure: the agent might still mutate files.
|
||||
func TestOwnActiveSessionKeepsRunUnsettled(t *testing.T) {
|
||||
_, r, pipelineID := agentStore(t)
|
||||
workspace := t.TempDir()
|
||||
ready := make(chan struct{})
|
||||
@@ -539,7 +538,7 @@ func TestOwnActiveSessionStillFailsRun(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "agent quiescence not confirmed") {
|
||||
t.Fatalf("own active session did not fail the run: %+v", detail)
|
||||
if detail.Status != "running" || !detail.CleanupUnconfirmed || detail.FinishedAt != "" {
|
||||
t.Fatalf("own active session falsely settled the run: %+v", detail)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestCancelWaitsForContextIgnoringExecutor(t *testing.T) {
|
||||
ctx, r, pipelineID := concurrentStore(t)
|
||||
entered, release := make(chan struct{}, 1), make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
stopExecutor := func() { releaseOnce.Do(func() { close(release) }) }
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
entered <- struct{}{}
|
||||
<-release // Deliberately ignore the canceled context.
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
defer func() { stopExecutor(); r.wg.Wait() }()
|
||||
workspace := t.TempDir()
|
||||
request := protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "blocking", Workspace: workspace, Input: map[string]any{"prompt": "task"}}
|
||||
run, err := r.start(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitEntered(t, entered)
|
||||
if err := r.cancel(run.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.cancel(run.ID); err != nil {
|
||||
t.Fatal("repeated stop must be idempotent", err)
|
||||
}
|
||||
detail, err := r.getRun(run.ID)
|
||||
if err != nil || detail.Status != "running" || !detail.Executing || !detail.CancelRequested || detail.FinishedAt != "" || detail.Steps[0].Status != "running" || detail.Steps[0].FinishedAt != "" {
|
||||
t.Fatalf("stop request falsely settled execution: %+v %v", detail, err)
|
||||
}
|
||||
request.RequestKey = "conflict"
|
||||
if _, err := r.start(request); err == nil {
|
||||
t.Fatal("stopping workspace was released")
|
||||
}
|
||||
// An actual second server must fail before recovery can touch this live row.
|
||||
other, err := pipeline.Open(filepath.Join(r.store.Directory(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer other.Close()
|
||||
if _, err := newRunner(ctx, other); err == nil || !strings.Contains(err.Error(), "another server owns") {
|
||||
t.Fatalf("second server did not fail before recovery: %v", err)
|
||||
}
|
||||
detail, _ = r.store.GetRun(ctx, run.ID)
|
||||
if detail.Status != "running" || detail.Steps[0].Status != "running" {
|
||||
t.Fatal("second server recovered live execution", detail)
|
||||
}
|
||||
stopExecutor()
|
||||
r.wg.Wait()
|
||||
detail, err = r.store.GetRun(ctx, run.ID)
|
||||
if err != nil || detail.Status != "canceled" || detail.Outcome != "" || detail.FinishedAt == "" || detail.Steps[0].Status != "canceled" {
|
||||
t.Fatalf("late success beat stop: %+v %v", detail, err)
|
||||
}
|
||||
r.mu.Lock()
|
||||
err = r.checkWorkspaceFree(workspace)
|
||||
r.mu.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal("confirmed stop did not release workspace", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupUnconfirmedReservesWorkspaceAcrossRecovery(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r, store, pipelineID := submitRunner(t, ctx, pipeline.Template([]string{"one"}, true))
|
||||
path := filepath.Join(store.Directory(), "test.db")
|
||||
defer func() { store.Close() }()
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{}, stage.ErrStageCleanup
|
||||
}
|
||||
workspace := t.TempDir()
|
||||
request := protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "cleanup", Workspace: workspace, Input: map[string]any{"prompt": "task"}}
|
||||
run, err := r.start(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
if err := r.cancel(run.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err := r.getRun(run.ID)
|
||||
if err != nil || detail.Status != "running" || !detail.CleanupUnconfirmed || detail.FinishedAt != "" || !detail.Executing {
|
||||
t.Fatalf("cleanup failure became terminal: %+v %v", detail, err)
|
||||
}
|
||||
request.RequestKey = "blocked"
|
||||
if _, err := r.start(request); err == nil {
|
||||
t.Fatal("unconfirmed cleanup released workspace")
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err = pipeline.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recovered, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// No recoverable session pointer exists in this fixture: fail closed rather
|
||||
// than claiming a restart proved an external process stopped.
|
||||
if _, err := recovered.start(request); err == nil {
|
||||
t.Fatal("restart released unconfirmed workspace without stop evidence")
|
||||
}
|
||||
detail, _ = recovered.getRun(run.ID)
|
||||
if !detail.CleanupUnconfirmed || detail.Status != "running" || detail.FinishedAt != "" {
|
||||
t.Fatal("recovery invented stop confirmation", detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceOverlapIncludesParentsDescendantsAndSymlinks(t *testing.T) {
|
||||
_, r, pipelineID := concurrentStore(t)
|
||||
workspace := t.TempDir()
|
||||
child := filepath.Join(workspace, "child")
|
||||
if err := os.Mkdir(child, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
alias := filepath.Join(t.TempDir(), "alias")
|
||||
if err := os.Symlink(child, alias); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{}, nil
|
||||
}
|
||||
// Admission resolves aliases; the overlap check must cover both directions.
|
||||
for _, pair := range [][2]string{{workspace, child}, {child, workspace}, {workspace, alias}} {
|
||||
r.mu.Lock()
|
||||
resolved, _ := filepath.EvalSymlinks(pair[0])
|
||||
r.active[99] = activeRun{cancel: func() {}, workspace: resolved}
|
||||
r.mu.Unlock()
|
||||
_, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "overlap", Workspace: pair[1], Input: map[string]any{"prompt": "task"}})
|
||||
if err == nil || !strings.Contains(err.Error(), "already active") {
|
||||
t.Fatal("overlapping workspace admission", pair, err)
|
||||
}
|
||||
}
|
||||
for _, pair := range [][2]string{{"/repo", "/repo-other"}, {"/repo/a", "/repo/b"}} {
|
||||
if workspacesOverlap(pair[0], pair[1]) {
|
||||
t.Fatal("sibling/prefix falsely overlaps", pair)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -102,6 +102,12 @@ func TestRetainedStageDiffsAcrossVisitsAndFailures(t *testing.T) {
|
||||
if ending == "cancel" {
|
||||
want = "canceled"
|
||||
}
|
||||
if ending == "cleanup" {
|
||||
want = "running"
|
||||
if !detail.CleanupUnconfirmed || detail.FinishedAt != "" {
|
||||
t.Fatal("unconfirmed cleanup reported as settled", detail)
|
||||
}
|
||||
}
|
||||
if detail.Status != want {
|
||||
t.Fatal("diff changed verdict", detail.Status, want)
|
||||
}
|
||||
|
||||
@@ -427,6 +427,8 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
|
||||
if in.NotifyMode != "" {
|
||||
into.NotifyMode = in.NotifyMode
|
||||
}
|
||||
into.CancelRequested = into.CancelRequested || in.CancelRequested
|
||||
into.CleanupUnconfirmed = into.CleanupUnconfirmed || in.CleanupUnconfirmed
|
||||
if in.Workspace != "" {
|
||||
into.Workspace = in.Workspace
|
||||
}
|
||||
@@ -503,19 +505,21 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
|
||||
continue
|
||||
}
|
||||
frame := protocol.RunEventFrame{
|
||||
NotifyMode: summary.NotifyMode,
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
Name: summary.Name,
|
||||
Title: summary.Title,
|
||||
Version: summary.Version,
|
||||
Status: summary.Status,
|
||||
Outcome: summary.Outcome,
|
||||
CurrentStage: summary.CurrentStage,
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
NotifyMode: summary.NotifyMode,
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
Name: summary.Name,
|
||||
Title: summary.Title,
|
||||
Version: summary.Version,
|
||||
Status: summary.Status,
|
||||
Outcome: summary.Outcome,
|
||||
CurrentStage: summary.CurrentStage,
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
CancelRequested: summary.CancelRequested,
|
||||
CleanupUnconfirmed: summary.CleanupUnconfirmed,
|
||||
}
|
||||
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
|
||||
// The connection write failed; the subscription is closed and the
|
||||
|
||||
+120
-42
@@ -23,8 +23,8 @@ import (
|
||||
"git.a-wan.net/alan/pipelines/internal/turn"
|
||||
)
|
||||
|
||||
// activeRun records one in-flight run's cancellation seam and the resolved
|
||||
// workspace it is changing. A single map is the only active-run state, so the
|
||||
// activeRun records an in-flight or cleanup-unconfirmed run's cancellation seam
|
||||
// and resolved workspace reservation. A single map is the active-run state, so the
|
||||
// workspace cannot drift from the cancel function.
|
||||
type activeRun struct {
|
||||
cancel context.CancelFunc
|
||||
@@ -77,6 +77,9 @@ type runner struct {
|
||||
}
|
||||
|
||||
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
if err := s.AcquireRunnerLock(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sessions, err := s.ActiveSessionOwners(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -101,7 +104,17 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
return nil, err
|
||||
}
|
||||
root := filepath.Join(s.Directory(), "artifacts")
|
||||
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
|
||||
active := map[int64]activeRun{}
|
||||
rows, err := s.ListRuns(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, row := range rows {
|
||||
if row.CleanupUnconfirmed {
|
||||
active[row.ID] = activeRun{cancel: func() {}, workspace: row.Workspace}
|
||||
}
|
||||
}
|
||||
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
|
||||
r.schedulerWG.Add(1)
|
||||
go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }()
|
||||
return r, nil
|
||||
@@ -331,24 +344,48 @@ func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
|
||||
go func() {
|
||||
defer r.wg.Done()
|
||||
defer cancel()
|
||||
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
|
||||
defer func() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
check, done := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer done()
|
||||
detail, err := r.store.GetRun(check, id)
|
||||
if err != nil || detail.CleanupUnconfirmed {
|
||||
return // Fail closed: execution/cleanup has not settled durably.
|
||||
}
|
||||
if detail.Status == protocol.RunStatusWaiting && detail.CancelRequested {
|
||||
if err := r.store.CancelWaiting(check, id); err != nil {
|
||||
return
|
||||
}
|
||||
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: detail.Workspace, Status: protocol.RunStatusCanceled, CancelRequested: true, CurrentStage: detail.CurrentStage, FinishedAt: time.Now().UTC().Format(time.RFC3339)}, true)
|
||||
}
|
||||
if detail.FinishedAt != "" || detail.Status == protocol.RunStatusWaiting {
|
||||
delete(r.active, id)
|
||||
}
|
||||
}()
|
||||
fn(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// checkWorkspaceFree refuses a run when an active run is already changing the
|
||||
// same resolved workspace. The workspace is the only scope a run reserves, and
|
||||
// one active run per workspace keeps this scan finding at most one match, so the
|
||||
// message is deterministic. Callers must hold r.mu.
|
||||
// checkWorkspaceFree refuses a run when an active or cleanup-unconfirmed run
|
||||
// reserves an overlapping resolved workspace. Callers must hold r.mu.
|
||||
func (r *runner) checkWorkspaceFree(workspace string) error {
|
||||
for id, entry := range r.active {
|
||||
if entry.workspace == workspace {
|
||||
if workspacesOverlap(entry.workspace, workspace) {
|
||||
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func workspacesOverlap(a, b string) bool {
|
||||
within := func(parent, child string) bool {
|
||||
rel, err := filepath.Rel(parent, child)
|
||||
return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator))
|
||||
}
|
||||
return within(a, b) || within(b, a)
|
||||
}
|
||||
|
||||
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
@@ -675,32 +712,34 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
|
||||
func (r *runner) cancel(id int64) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
entry, ok := r.active[id]
|
||||
r.mu.Unlock()
|
||||
// Persist the cancellation before asking the runner to stop. A stage that is
|
||||
// stuck in a context-unaware provider call must not leave the run "running":
|
||||
// the operator's Stop is a terminal decision, and the live row/step show it
|
||||
// immediately. FinishStep/FinishRun only update active rows, so the late
|
||||
// stage return cannot undo this.
|
||||
if err := r.store.Cancel(r.ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
entry.cancel()
|
||||
} else {
|
||||
// Only a parked human run is known to have no live executor.
|
||||
if err := r.store.CancelWaiting(r.ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
r.forgetAttention(id)
|
||||
if r.events != nil {
|
||||
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil {
|
||||
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
|
||||
r.emitRun(protocol.RunEventFrame{
|
||||
RunID: id,
|
||||
Workspace: detail.Workspace,
|
||||
Status: detail.Status,
|
||||
Outcome: detail.Outcome,
|
||||
CurrentStage: detail.CurrentStage,
|
||||
CompletedSteps: detail.CompletedSteps,
|
||||
FinishedAt: detail.FinishedAt,
|
||||
}, true)
|
||||
RunID: id,
|
||||
Workspace: detail.Workspace,
|
||||
Status: detail.Status,
|
||||
Outcome: detail.Outcome,
|
||||
CurrentStage: detail.CurrentStage,
|
||||
CompletedSteps: detail.CompletedSteps,
|
||||
FinishedAt: detail.FinishedAt,
|
||||
CancelRequested: detail.CancelRequested,
|
||||
CleanupUnconfirmed: detail.CleanupUnconfirmed,
|
||||
}, !ok)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -1265,20 +1304,22 @@ func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
|
||||
}
|
||||
|
||||
func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) {
|
||||
r.mu.Lock()
|
||||
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
|
||||
// Cancel wins over a late stage result: the store already marked the run
|
||||
// canceled and FinishRun only updates queued/running rows. Read the
|
||||
// actual state so this turn emits and notifies canceled, never a failure
|
||||
// caused by the canceled step's persistence guard.
|
||||
if detail, getErr := r.store.GetRun(c, id); getErr == nil && detail != nil && detail.Status == protocol.RunStatusCanceled {
|
||||
status = protocol.RunStatusCanceled
|
||||
outcome = detail.Outcome
|
||||
diagnostic = detail.Diagnostic
|
||||
} else {
|
||||
log.Printf("finish run %d: %v", id, err)
|
||||
}
|
||||
log.Printf("finish run %d: %v", id, err)
|
||||
}
|
||||
// The store atomically makes a stop request win over a late success.
|
||||
detail, getErr := r.store.GetRun(c, id)
|
||||
if getErr != nil || detail == nil {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
status, outcome, diagnostic = detail.Status, detail.Outcome, detail.Diagnostic
|
||||
r.mu.Unlock()
|
||||
if detail.FinishedAt == "" {
|
||||
return // Cleanup is not confirmed: never emit a false terminal event.
|
||||
}
|
||||
// Push the terminal delta before the attention notification, so a slow ntfy
|
||||
// never delays the live update the app folds.
|
||||
@@ -1289,7 +1330,7 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
|
||||
Outcome: outcome,
|
||||
CurrentStage: currentStage,
|
||||
CompletedSteps: completed,
|
||||
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
FinishedAt: detail.FinishedAt,
|
||||
}, true)
|
||||
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
|
||||
// The run is terminal: nothing more can notify for it.
|
||||
@@ -1353,12 +1394,27 @@ func turnOutputOnly(output stage.Values) bool {
|
||||
}
|
||||
|
||||
func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) {
|
||||
r.mu.Lock()
|
||||
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
detail, getErr := r.store.GetRun(save, id)
|
||||
if getErr != nil {
|
||||
cancel()
|
||||
r.mu.Unlock()
|
||||
return "", "", "", false, getErr
|
||||
}
|
||||
if detail.CancelRequested {
|
||||
step.Status, step.Outcome, step.Target = "canceled", "", ""
|
||||
step.Diagnostic = "Canceled by operator; execution returned"
|
||||
}
|
||||
persistErr := r.store.FinishStep(save, id, step)
|
||||
cancel()
|
||||
r.mu.Unlock()
|
||||
if persistErr != nil {
|
||||
return "", "", "", false, persistErr
|
||||
}
|
||||
if detail.CancelRequested {
|
||||
return "canceled", "", step.Diagnostic, true, nil
|
||||
}
|
||||
if step.Status == "completed" {
|
||||
values, copyErr := copyValues(stage.Values(step.Output))
|
||||
if copyErr != nil {
|
||||
@@ -1696,6 +1752,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
result, err = stage.DefaultRegistry().Run(stageCtx, def, input)
|
||||
}
|
||||
}()
|
||||
ownershipCtx, ownershipCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
activeSessions, ownershipErr := r.store.ActiveAgentSessionsForRun(ownershipCtx, id)
|
||||
ownershipCancel()
|
||||
if ownershipErr != nil || len(activeSessions) > 0 {
|
||||
err = errors.Join(stage.ErrStageCleanup, err, ownershipErr)
|
||||
}
|
||||
if ctx.Err() != nil && !errors.Is(err, stage.ErrStageCleanup) {
|
||||
err = ctx.Err() // Ignore a context-unaware executor's late success.
|
||||
}
|
||||
if stage.EffectiveType(def.Type) == stage.ModelType && err == nil {
|
||||
ensureTurnOutput(visitTurn, result)
|
||||
}
|
||||
@@ -1707,6 +1772,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
if errors.Is(err, stage.ErrStageCleanup) {
|
||||
changes.Status = "partial"
|
||||
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
|
||||
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if markErr := r.store.MarkCleanupUnconfirmed(save, id); markErr != nil {
|
||||
log.Printf("mark cleanup unconfirmed for run %d: %v", id, markErr)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
// Export the stage's session into its own artifacts directory before the
|
||||
// bounded scan, so the existing listing and serving paths find the files
|
||||
@@ -1790,6 +1860,17 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
|
||||
}
|
||||
step.Target = def.Edges[step.Outcome]
|
||||
if errors.Is(err, stage.ErrStageCleanup) {
|
||||
// Retain the partial evidence without pretending that the agent stopped.
|
||||
step.Status, step.Outcome, step.Target = "running", "", ""
|
||||
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if saveErr := r.store.FinishStep(save, id, step); saveErr != nil {
|
||||
log.Printf("save cleanup evidence for run %d: %v", id, saveErr)
|
||||
}
|
||||
cancel()
|
||||
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CleanupUnconfirmed: true}, false)
|
||||
return
|
||||
}
|
||||
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
|
||||
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
|
||||
for _, outcome := range selected.Limits.On {
|
||||
@@ -1823,6 +1904,10 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
finish("failed", "error", finishErr.Error())
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
return
|
||||
}
|
||||
// Advance the in-memory UI-surface evidence only after the step is
|
||||
// persisted, so the skip guard agrees with what the store holds.
|
||||
state.recordChanges(step.FileChanges, step.Artifacts)
|
||||
@@ -1856,13 +1941,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
Artifacts: len(step.Artifacts),
|
||||
},
|
||||
}, false)
|
||||
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
|
||||
sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id)
|
||||
if checkErr != nil || len(sessions) > 0 {
|
||||
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
|
||||
return
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
return
|
||||
|
||||
@@ -135,7 +135,12 @@ func TestHandlerWiresRunTitleGenerator(t *testing.T) {
|
||||
}
|
||||
defer store.Close()
|
||||
configured := HandlerWithOptions(ctx, Options{RunTitles: summarise.GenerateTitle}, store).(*runHandler)
|
||||
bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler)
|
||||
bareStore, err := pipeline.Open(filepath.Join(t.TempDir(), "bare.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer bareStore.Close()
|
||||
bare := HandlerWithOptions(ctx, Options{}, bareStore).(*runHandler)
|
||||
defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }()
|
||||
if configured.runner.title == nil {
|
||||
t.Fatal("configured RunTitles was not applied to the runner")
|
||||
@@ -461,7 +466,12 @@ func TestHandlerWiresStepVerdictGenerator(t *testing.T) {
|
||||
}
|
||||
defer store.Close()
|
||||
configured := HandlerWithOptions(ctx, Options{StepVerdicts: summarise.GenerateVerdictReason}, store).(*runHandler)
|
||||
bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler)
|
||||
bareStore, err := pipeline.Open(filepath.Join(t.TempDir(), "bare.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer bareStore.Close()
|
||||
bare := HandlerWithOptions(ctx, Options{}, bareStore).(*runHandler)
|
||||
defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }()
|
||||
if configured.runner.verdict == nil {
|
||||
t.Fatal("configured StepVerdicts was not applied to the runner")
|
||||
|
||||
@@ -71,7 +71,7 @@ func sessionExportRunner(t *testing.T, backend string, seam func(stage.SessionEx
|
||||
var runID int64
|
||||
handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
<-ready
|
||||
if err := handler.runner.store.RunSession(ctx, runID, 1, "ses_run", true, backend, ""); err != nil {
|
||||
if err := handler.runner.store.RunSession(ctx, runID, 1, "ses_run", false, backend, ""); err != nil {
|
||||
return stage.Result{}, err
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
|
||||
|
||||
+1
-1
@@ -154,7 +154,7 @@ export class DecisionDraft {
|
||||
|
||||
/** Cancel is offered only while the run is executing. */
|
||||
export function canCancel(run) {
|
||||
return Boolean(run?.executing);
|
||||
return Boolean(run?.executing) && !run.cancel_requested && !run.cleanup_unconfirmed;
|
||||
}
|
||||
|
||||
/** Retry is offered only for a loaded, non-executing, `failed` run. */
|
||||
|
||||
+11
-1
@@ -181,6 +181,8 @@ function runStateFields(current) {
|
||||
created_at: current.created_at ?? "",
|
||||
finished_at: current.finished_at ?? "",
|
||||
executing: Boolean(current.executing),
|
||||
cancel_requested: Boolean(current.cancel_requested),
|
||||
cleanup_unconfirmed: Boolean(current.cleanup_unconfirmed),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -210,6 +212,8 @@ function applyRunStateFields(current, frame) {
|
||||
created_at: blank(frame.created_at) ? before.created_at : frame.created_at,
|
||||
finished_at: blank(frame.finished_at) ? before.finished_at : frame.finished_at,
|
||||
executing: executingFor(status, before.executing),
|
||||
cancel_requested: before.cancel_requested || frame.cancel_requested === true,
|
||||
cleanup_unconfirmed: before.cleanup_unconfirmed || frame.cleanup_unconfirmed === true,
|
||||
};
|
||||
const changed =
|
||||
merged.notify_mode !== before.notify_mode ||
|
||||
@@ -220,7 +224,9 @@ function applyRunStateFields(current, frame) {
|
||||
merged.completed_steps !== before.completed_steps ||
|
||||
merged.created_at !== before.created_at ||
|
||||
merged.finished_at !== before.finished_at ||
|
||||
merged.executing !== before.executing;
|
||||
merged.executing !== before.executing ||
|
||||
merged.cancel_requested !== before.cancel_requested ||
|
||||
merged.cleanup_unconfirmed !== before.cleanup_unconfirmed;
|
||||
return changed ? { ...current, ...merged } : current;
|
||||
}
|
||||
|
||||
@@ -244,6 +250,8 @@ export function newRunSummaryFromFrame(frame) {
|
||||
version: Number(frame.version) || 0,
|
||||
status: frame.status ?? "",
|
||||
executing: executingFor(frame.status, true),
|
||||
cancel_requested: frame.cancel_requested === true,
|
||||
cleanup_unconfirmed: frame.cleanup_unconfirmed === true,
|
||||
outcome: frame.outcome ?? "",
|
||||
current_stage: frame.current_stage ?? "",
|
||||
created_at: frame.created_at ?? "",
|
||||
@@ -361,6 +369,8 @@ export function runDetailNeedsFullSync(frame) {
|
||||
return (
|
||||
stepSettled ||
|
||||
stepWaiting ||
|
||||
frame?.cancel_requested === true ||
|
||||
frame?.cleanup_unconfirmed === true ||
|
||||
isTerminalRunStatus(frame?.status) ||
|
||||
isWaitingRunStatus(frame?.status)
|
||||
);
|
||||
|
||||
@@ -37,6 +37,14 @@ export const RUN_STATES = [
|
||||
* states so one chip set renders list, detail and timeline). Mirrors
|
||||
* runStateOf: blank or unrecognised input is the safe `unknown` fallback.
|
||||
*/
|
||||
export function runStatusLabel(run) {
|
||||
if (["queued", "running", "waiting"].includes(run?.status)) {
|
||||
if (run.cleanup_unconfirmed) return "Cleanup unconfirmed — workspace reserved";
|
||||
if (run.cancel_requested) return "Stopping — waiting for execution to stop";
|
||||
}
|
||||
return run?.status || "unknown";
|
||||
}
|
||||
|
||||
export function runStateOf(status) {
|
||||
switch (String(status ?? "").trim().toLowerCase()) {
|
||||
case "running":
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
import { h, kvMono, canCopyLink, canShare, copyArtifactLink, shareArtifact } from "../dom.js";
|
||||
import { isExecuting, changeKindTone, describeError } from "../format.js";
|
||||
import { resolveArtifactUrl } from "../serverUrl.js";
|
||||
import { runStateOf } from "../runs.js";
|
||||
import { runStateOf, runStatusLabel } from "../runs.js";
|
||||
import { PIPELINE_GRAPH_EMPTY, RUN_GRAPH_MISSING, mount as mountGraph } from "../graph.js";
|
||||
import { writeGraphExpanded } from "../graphPrefs.js";
|
||||
import {
|
||||
@@ -135,11 +135,11 @@ function renderRunDetail(now = Date.now()) {
|
||||
if (decision) pane.append(renderDecisionCard(decision));
|
||||
const status = runStateOf(d.status);
|
||||
const kicker = h("div", { class: "run-kicker" });
|
||||
kicker.append(h("span", { class: `chip chip-${status}`, text: d.status || status }));
|
||||
kicker.append(h("span", { class: `chip chip-${status}`, text: runStatusLabel(d) }));
|
||||
if (String(d.outcome ?? "").trim()) {
|
||||
kicker.append(h("span", { class: "muted", text: `outcome ${d.outcome}` }));
|
||||
}
|
||||
if (d.executing) kicker.append(h("span", { class: "muted", text: "executing" }));
|
||||
if (d.executing && !d.cleanup_unconfirmed) kicker.append(h("span", { class: "muted", text: "executing" }));
|
||||
pane.append(kicker);
|
||||
pane.append(
|
||||
h("h1", {
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
RUN_TIME_RANGES,
|
||||
RUN_SORTS,
|
||||
runStateOf,
|
||||
runStatusLabel,
|
||||
runIdentity,
|
||||
versionLabel,
|
||||
progressText,
|
||||
@@ -125,7 +126,10 @@ function renderRunRow(run, now) {
|
||||
});
|
||||
|
||||
const head = h("span", { class: "run-row-head" });
|
||||
head.append(statusIcon(status, status.charAt(0).toUpperCase() + status.slice(1)));
|
||||
head.append(statusIcon(status, runStatusLabel(run)));
|
||||
if (run.cancel_requested || run.cleanup_unconfirmed) {
|
||||
head.append(h("span", { class: "muted", text: runStatusLabel(run) }));
|
||||
}
|
||||
if (run.executing) head.append(h("span", { class: "spinner", "aria-hidden": "true" }));
|
||||
// The pipeline is the list's anchor: it names what ran, before the task title.
|
||||
head.append(
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { canCancel } from "../src/actions.js";
|
||||
import { runStatusLabel } from "../src/runs.js";
|
||||
import { foldRunEvent, foldRunEventDetail } from "../src/runEvents.js";
|
||||
|
||||
test("stop intent stays nonterminal, disables Stop, and folds into live rows", () => {
|
||||
const run = { id: 4, status: "running", executing: true, steps: [] };
|
||||
const frame = { run_id: 4, status: "running", cancel_requested: true };
|
||||
const updated = foldRunEvent([run], frame).runs[0];
|
||||
assert.equal(updated.status, "running");
|
||||
assert.equal(updated.executing, true);
|
||||
assert.equal(canCancel(updated), false);
|
||||
assert.match(runStatusLabel(updated), /Stopping/);
|
||||
assert.equal(foldRunEventDetail(run, frame).cancel_requested, true);
|
||||
});
|
||||
|
||||
test("cleanup uncertainty remains visible until a terminal observation", () => {
|
||||
const run = { id: 4, status: "running", executing: true };
|
||||
const pending = foldRunEvent([run], { run_id: 4, cleanup_unconfirmed: true }).runs[0];
|
||||
assert.equal(canCancel(pending), false);
|
||||
assert.match(runStatusLabel(pending), /Cleanup unconfirmed.*workspace reserved/);
|
||||
const terminal = foldRunEvent([pending], { run_id: 4, status: "canceled" }).runs[0];
|
||||
assert.equal(terminal.executing, false);
|
||||
assert.equal(runStatusLabel(terminal), "canceled");
|
||||
});
|
||||
Reference in New Issue
Block a user