diff --git a/docs/cli.md b/docs/cli.md index 73b3e786..5dae6a7e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -100,9 +100,12 @@ Every submission, including agent-issued `submit`/`rpc`, still uses server valid ## Drafts in the TUI -The server admits one active run at a time. A new request key is rejected while -another run is active; inspect that run and wait for it to finish. Reusing its -original request key reattaches idempotently. +The server runs one active run per resolved workspace. A new request key in a +different workspace is admitted immediately and executes concurrently; a second +run in a workspace an active run is already changing is refused, naming the +conflicting run and advising you to wait or use a different workspace (for +example another checkout of the repository). Reusing the original request key +reattaches idempotently. Use `:templates` for five simple, manual-only starters: task, summary, review, research, and plan/implement/review. **Submit template** (`s` or Enter) asks for a diff --git a/docs/first-version-testing.md b/docs/first-version-testing.md index d6878bac..90e75537 100644 --- a/docs/first-version-testing.md +++ b/docs/first-version-testing.md @@ -95,7 +95,9 @@ delivery to your phone. A summary starter does not send alerts automatically. ## Known boundaries -- Sequential stages only; no parallel execution or built-in human-approval stages. +- Sequential stages within a run; no parallel branches or built-in + human-approval stages. Independent runs in different workspaces do execute + concurrently. - Cancellation/retry is **not rollback**. Retry starts from the entry stage with fresh sessions and the pinned definition; earlier side effects can repeat. - Diffs are bounded stage-start/stage-end observations, not full backups or proof diff --git a/internal/author/skill/references/operations.md b/internal/author/skill/references/operations.md index 0ce54ee1..52dfc044 100644 --- a/internal/author/skill/references/operations.md +++ b/internal/author/skill/references/operations.md @@ -21,13 +21,15 @@ parallel/distributed task engine or a guarantee that model reports are correct. client does not upload its repository or make host credentials available remotely. - Host and Docker OpenCode credentials are separate. Preserve setup-selected model inheritance; do not select a provider just because its ID appears in an example. -- Runtime caps are 30 stage steps and five visits to a stage. There are no parallel - runs on one server: admission rejects new work while another run is active; - reusing that active run's request key reattaches idempotently. Inspect and wait - for the existing run rather than starting duplicates. There are no parallel - branches or built-in human approval stages. Optional worktrees, full backup/restore, - automatic database sync and independent verifier stage types are future ideas, - not existing capabilities. Do not invent them in definitions. +- Runtime caps are 30 stage steps and five visits to a stage. Runs in different + resolved workspaces execute concurrently; only a second run in a workspace an + active run is already changing is refused, naming the conflicting run and + advising you to wait or use a different workspace (for example another + checkout). Reusing that active run's request key reattaches idempotently. + There are no parallel branches or built-in human approval stages. Optional + worktrees, full backup/restore, automatic database sync and independent + verifier stage types are future ideas, not existing capabilities. Do not + invent them in definitions. - Services are native or Docker-based. With an existing Docker installation do not start a competing native server. Server APIs are unauthenticated: trusted networks only; never recommend public exposure. Tool scope is not a universal OS sandbox. diff --git a/internal/server/concurrency_test.go b/internal/server/concurrency_test.go new file mode 100644 index 00000000..d99b1dbc --- /dev/null +++ b/internal/server/concurrency_test.go @@ -0,0 +1,346 @@ +package server + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func concurrentStore(t *testing.T) (context.Context, *runner, int64) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + store, err := pipeline.Open(filepath.Join(t.TempDir(), "concurrent.db")) + if err != nil { + cancel() + t.Fatal(err) + } + if _, err := store.Submit(ctx, "test", pipeline.Template([]string{"one"}, true)); err != nil { + cancel() + store.Close() + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + cancel() + store.Close() + t.Fatal(err) + } + r, err := newRunner(ctx, store) + if err != nil { + cancel() + store.Close() + t.Fatal(err) + } + t.Cleanup(func() { + cancel() + r.wg.Wait() + store.Close() + }) + return ctx, r, items[0].ID +} + +// waitEntered fails fast instead of hanging when a stage never starts. +func waitEntered(t *testing.T, entered <-chan struct{}) { + t.Helper() + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("stage execution did not start") + } +} + +// blockingExecute is the r.execute seam for these tests: it signals each entry +// and then blocks until release is closed, so a test can hold several real runs +// executing at once and observe them through getRun/listRuns. +func blockingExecute(entered chan<- struct{}, release <-chan struct{}) func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { + entered <- struct{}{} + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } +} + +// TestDifferentWorkspacesRunConcurrently proves two runs in two different +// resolved workspaces are admitted at the same time and both report executing. +func TestDifferentWorkspacesRunConcurrently(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + a, b := t.TempDir(), t.TempDir() + entered := make(chan struct{}, 2) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatalf("second workspace was blocked by the first: %v", err) + } + waitEntered(t, entered) + for _, id := range []int64{first.ID, second.ID} { + detail, err := r.getRun(id) + if err != nil || !detail.Executing { + t.Fatalf("run %d was not reported executing: %v", id, err) + } + } + runs, err := r.listRuns() + if err != nil { + t.Fatal(err) + } + executing := map[int64]bool{} + for _, run := range runs { + executing[run.ID] = run.Executing + } + if !executing[first.ID] || !executing[second.ID] { + t.Fatalf("listRuns did not report both runs executing: %+v", runs) + } + close(release) + r.wg.Wait() + for _, id := range []int64{first.ID, second.ID} { + detail, err := r.getRun(id) + if err != nil || detail.Status != "succeeded" || detail.Executing { + t.Fatalf("run %d did not reach succeeded: %+v %v", id, detail, err) + } + } +} + +// TestSameWorkspaceSecondRunIsRefused proves a different request key in the same +// resolved workspace is refused, naming the conflicting run and the remedy. +func TestSameWorkspaceSecondRunIsRefused(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + workspace := t.TempDir() + entered := make(chan struct{}, 2) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "first", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "second", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) || !strings.Contains(err.Error(), "different workspace") { + t.Fatalf("same-workspace run was not refused with an actionable conflict: %v", err) + } + close(release) + r.wg.Wait() +} + +// TestConcurrentSameWorkspaceStartsRefuseAllButOne proves the check and the +// registration of r.active are atomic under r.mu: two simultaneous starts for +// one workspace cannot both succeed. +func TestConcurrentSameWorkspaceStartsRefuseAllButOne(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + workspace := t.TempDir() + entered := make(chan struct{}, 2) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + results := make(chan error, 2) + var wg sync.WaitGroup + for _, key := range []string{"one", "two"} { + wg.Add(1) + go func(key string) { + defer wg.Done() + _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: key, Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + results <- err + }(key) + } + wg.Wait() + close(results) + refused := 0 + for err := range results { + if err != nil { + refused++ + } + } + if refused != 1 { + t.Fatalf("expected exactly one refusal for one workspace, got %d", refused) + } + waitEntered(t, entered) + close(release) + r.wg.Wait() +} + +// TestSymlinkedWorkspaceConflictsWithItsTarget proves two spellings of one +// directory (symlink vs real path) resolve equal and conflict. +func TestSymlinkedWorkspaceConflictsWithItsTarget(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + root := t.TempDir() + real := filepath.Join(root, "repo") + if err := os.Mkdir(real, 0700); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "linked-repo") + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + entered := make(chan struct{}, 2) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "real", Workspace: real, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "link", Workspace: link, Input: map[string]any{"prompt": "task"}}) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) { + t.Fatalf("symlink and target did not resolve to the same workspace: %v", err) + } + close(release) + r.wg.Wait() +} + +// TestDefaultAndExplicitWorkspaceConflict proves the default cwd and an +// explicit absolute spelling of the same directory resolve equal and conflict. +func TestDefaultAndExplicitWorkspaceConflict(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + entered := make(chan struct{}, 2) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: cwd, Input: map[string]any{"prompt": "task"}}) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) { + t.Fatalf("default cwd and its explicit absolute path did not conflict: %v", err) + } + close(release) + r.wg.Wait() +} + +// TestIdempotentRepeatWhileActive proves repeating one request key returns the +// same active run whether the run is in an explicit workspace or the default. +func TestIdempotentRepeatWhileActive(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + entered := make(chan struct{}, 4) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + explicit := t.TempDir() + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + repeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}}) + if err != nil || repeat.ID != first.ID || !repeat.Executing { + t.Fatalf("explicit-workspace repeat was not idempotent: %+v %v", repeat, err) + } + // Default workspace: no Workspace field, resolved from the process cwd. + defaultFirst, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + defaultRepeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}) + if err != nil || defaultRepeat.ID != defaultFirst.ID || !defaultRepeat.Executing { + t.Fatalf("default-workspace repeat was not idempotent: %+v %v", defaultRepeat, err) + } + close(release) + r.wg.Wait() +} + +// TestRestartRecoveryStillRefuses proves the existing guard is unchanged: an +// agent session left active with no in-memory active run blocks new work. +func TestRestartRecoveryStillRefuses(t *testing.T) { + ctx, r, pipelineID := concurrentStore(t) + d, err := r.store.Get(ctx, pipelineID, 1) + if err != nil { + t.Fatal(err) + } + workspace := t.TempDir() + id, _, err := r.store.CreateRun(ctx, d, "leftover", workspace, false, nil, map[string]any{"prompt": "task"}) + if err != nil { + t.Fatal(err) + } + if err := r.store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "task"}); err != nil { + t.Fatal(err) + } + if err := r.store.RunSession(ctx, id, 1, "ses_leftover", true, "v2", ""); err != nil { + t.Fatal(err) + } + if len(r.active) != 0 { + t.Fatal("test expected no in-memory active runs") + } + _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "new", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + if err == nil || !strings.Contains(err.Error(), "restart recovery must interrupt it") { + t.Fatalf("restart-recovery refusal was not preserved: %v", err) + } +} + +// TestRetryConflictsOnlyWithSameWorkspaceActiveRun proves retry forwards the +// original run's workspace and is refused only by an active run in that same +// workspace. +func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) { + ctx, r, pipelineID := concurrentStore(t) + a, b := t.TempDir(), t.TempDir() + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{}, fmt.Errorf("deliberate failure") + } + original, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "original", Workspace: a, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + failed, err := r.store.GetRun(ctx, original.ID) + if err != nil || failed.Status != "failed" { + t.Fatalf("original run did not fail as intended: %+v %v", failed, err) + } + resolvedB, err := filepath.EvalSymlinks(b) + if err != nil { + t.Fatal(err) + } + entered := make(chan struct{}, 4) + release := make(chan struct{}) + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + entered <- struct{}{} + if input["pwd"] == resolvedB { + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + activeA, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-a", Workspace: a, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + if _, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-a"}); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", activeA.ID)) || !strings.Contains(err.Error(), "different workspace") { + t.Fatalf("retry did not conflict with same-workspace active run: %v", err) + } + // A different active workspace must not block the retry of the original. + if _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-b", Workspace: b, Input: map[string]any{"prompt": "task"}}); err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-b"}) + if err != nil { + t.Fatalf("retry was blocked by an unrelated workspace: %v", err) + } + if retried.Workspace != failed.Workspace { + t.Fatalf("retry did not reuse the original workspace: %q != %q", retried.Workspace, failed.Workspace) + } + close(release) + r.wg.Wait() +} diff --git a/internal/server/retry_cleanup_test.go b/internal/server/retry_cleanup_test.go index 75fb2ad9..ffe00230 100644 --- a/internal/server/retry_cleanup_test.go +++ b/internal/server/retry_cleanup_test.go @@ -34,7 +34,7 @@ func TestRetryWaitsForExecutionCleanupEvenAfterTerminalStatus(t *testing.T) { t.Fatal(err) } // Simulate the interval between persisting terminal status and final cleanup. - r.active[id] = func() {} + r.active[id] = activeRun{cancel: func() {}} detail, err := r.getRun(id) if err != nil || !detail.Executing { t.Fatal("terminal run not marked executing during cleanup", detail, err) diff --git a/internal/server/runner.go b/internal/server/runner.go index 10c7c2b5..abb2f992 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -19,11 +19,19 @@ import ( "git.a-wan.net/alan/pipelines/internal/stage" ) +// activeRun records one in-flight run's cancellation seam and the resolved +// workspace it is changing. A single map is the only active-run state, so the +// workspace cannot drift from the cancel function. +type activeRun struct { + cancel context.CancelFunc + workspace string +} + type runner struct { ctx context.Context store *pipeline.Store mu sync.Mutex - active map[int64]context.CancelFunc + active map[int64]activeRun execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error) wg sync.WaitGroup schedulerWG sync.WaitGroup @@ -50,7 +58,7 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { if err := s.RecoverRuns(ctx); err != nil { return nil, err } - return &runner{ctx: ctx, store: s, active: map[int64]context.CancelFunc{}}, nil + return &runner{ctx: ctx, store: s, active: map[int64]activeRun{}}, nil } func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) { @@ -166,7 +174,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru return nil, err } if existing == 0 { - return nil, fmt.Errorf("another run is active; inspect it before starting more work") + // The workspace is the only scope a run reserves: refuse a second + // run only when an active run is already changing this exact + // resolved directory. One active run per workspace keeps this scan + // finding at most one match, so the message is deterministic. + for id, entry := range r.active { + if entry.workspace == workspace { + return nil, fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace) + } + } } } id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, retryOf...) @@ -175,7 +191,7 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru } if created { ctx, cancel := context.WithCancel(r.ctx) - r.active[id] = cancel + r.active[id] = activeRun{cancel: cancel, workspace: workspace} r.wg.Add(1) go func() { defer r.wg.Done() @@ -247,11 +263,11 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { func (r *runner) cancel(id int64) error { r.mu.Lock() defer r.mu.Unlock() - cancel, ok := r.active[id] + entry, ok := r.active[id] if !ok { return fmt.Errorf("run is not active") } - cancel() + entry.cancel() return nil } diff --git a/internal/server/runner_test.go b/internal/server/runner_test.go index f9ba97e6..2da8d9fe 100644 --- a/internal/server/runner_test.go +++ b/internal/server/runner_test.go @@ -3,6 +3,7 @@ package server import ( "context" "errors" + "fmt" "path/filepath" "strings" "testing" @@ -56,8 +57,8 @@ func TestRunnerRoutingPersistenceAndIdempotency(t *testing.T) { } other := request other.RequestKey = "other" - if _, err := r.start(other); err == nil { - t.Fatal("overlapping execution accepted") + if _, err := r.start(other); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", run.ID)) || !strings.Contains(err.Error(), "different workspace") { + t.Fatal("overlapping execution was not refused with the conflicting run", err) } close(release) r.wg.Wait()