From 2618f0d2448be4e3d1717ccf19d2679ed881b910 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Thu, 15 Oct 2026 15:08:44 +0100 Subject: [PATCH] Allow concurrent runs in different workspaces Track each active run's resolved workspace alongside its cancel function in a single runner.active struct map. A new request key is now refused only when an already-active run is changing the same resolved workspace; the refusal names the conflicting run id and advises waiting or using a different workspace. Workspace resolution is unchanged (default cwd, absolute request workspace, EvalSymlinks), so two spellings of one directory still compare equal. The conflict check and registration stay inside the existing r.mu critical section, so concurrent starts for one workspace cannot both pass. Idempotent repeats of an active request key and the restart-recovery guard are unchanged. Add concurrency_test.go covering concurrent different-workspace runs, same-workspace refusal, symlink/default-cwd equivalence, idempotent repeats, restart recovery and retry workspace scoping; strengthen the runner_test.go refusal assertion. Update docs/cli.md, docs/first-version-testing.md and the packaged operations skill reference to describe per-workspace concurrency. Assisted-by: DeepSeek V4.1 Flash --- docs/cli.md | 9 +- docs/first-version-testing.md | 4 +- .../author/skill/references/operations.md | 16 +- internal/server/concurrency_test.go | 346 ++++++++++++++++++ internal/server/retry_cleanup_test.go | 2 +- internal/server/runner.go | 28 +- internal/server/runner_test.go | 5 +- 7 files changed, 390 insertions(+), 20 deletions(-) create mode 100644 internal/server/concurrency_test.go diff --git a/docs/cli.md b/docs/cli.md index 73b3e786..5dae6a7e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -100,9 +100,12 @@ Every submission, including agent-issued `submit`/`rpc`, still uses server valid ## Drafts in the TUI -The server admits one active run at a time. A new request key is rejected while -another run is active; inspect that run and wait for it to finish. Reusing its -original request key reattaches idempotently. +The server runs one active run per resolved workspace. A new request key in a +different workspace is admitted immediately and executes concurrently; a second +run in a workspace an active run is already changing is refused, naming the +conflicting run and advising you to wait or use a different workspace (for +example another checkout of the repository). Reusing the original request key +reattaches idempotently. Use `:templates` for five simple, manual-only starters: task, summary, review, research, and plan/implement/review. **Submit template** (`s` or Enter) asks for a diff --git a/docs/first-version-testing.md b/docs/first-version-testing.md index d6878bac..90e75537 100644 --- a/docs/first-version-testing.md +++ b/docs/first-version-testing.md @@ -95,7 +95,9 @@ delivery to your phone. A summary starter does not send alerts automatically. ## Known boundaries -- Sequential stages only; no parallel execution or built-in human-approval stages. +- Sequential stages within a run; no parallel branches or built-in + human-approval stages. Independent runs in different workspaces do execute + concurrently. - Cancellation/retry is **not rollback**. Retry starts from the entry stage with fresh sessions and the pinned definition; earlier side effects can repeat. - Diffs are bounded stage-start/stage-end observations, not full backups or proof diff --git a/internal/author/skill/references/operations.md b/internal/author/skill/references/operations.md index 0ce54ee1..52dfc044 100644 --- a/internal/author/skill/references/operations.md +++ b/internal/author/skill/references/operations.md @@ -21,13 +21,15 @@ parallel/distributed task engine or a guarantee that model reports are correct. client does not upload its repository or make host credentials available remotely. - Host and Docker OpenCode credentials are separate. Preserve setup-selected model inheritance; do not select a provider just because its ID appears in an example. -- Runtime caps are 30 stage steps and five visits to a stage. There are no parallel - runs on one server: admission rejects new work while another run is active; - reusing that active run's request key reattaches idempotently. Inspect and wait - for the existing run rather than starting duplicates. There are no parallel - branches or built-in human approval stages. Optional worktrees, full backup/restore, - automatic database sync and independent verifier stage types are future ideas, - not existing capabilities. Do not invent them in definitions. +- Runtime caps are 30 stage steps and five visits to a stage. Runs in different + resolved workspaces execute concurrently; only a second run in a workspace an + active run is already changing is refused, naming the conflicting run and + advising you to wait or use a different workspace (for example another + checkout). Reusing that active run's request key reattaches idempotently. + There are no parallel branches or built-in human approval stages. Optional + worktrees, full backup/restore, automatic database sync and independent + verifier stage types are future ideas, not existing capabilities. Do not + invent them in definitions. - Services are native or Docker-based. With an existing Docker installation do not start a competing native server. Server APIs are unauthenticated: trusted networks only; never recommend public exposure. Tool scope is not a universal OS sandbox. diff --git a/internal/server/concurrency_test.go b/internal/server/concurrency_test.go new file mode 100644 index 00000000..d99b1dbc --- /dev/null +++ b/internal/server/concurrency_test.go @@ -0,0 +1,346 @@ +package server + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func concurrentStore(t *testing.T) (context.Context, *runner, int64) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + store, err := pipeline.Open(filepath.Join(t.TempDir(), "concurrent.db")) + if err != nil { + cancel() + t.Fatal(err) + } + if _, err := store.Submit(ctx, "test", pipeline.Template([]string{"one"}, true)); err != nil { + cancel() + store.Close() + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + cancel() + store.Close() + t.Fatal(err) + } + r, err := newRunner(ctx, store) + if err != nil { + cancel() + store.Close() + t.Fatal(err) + } + t.Cleanup(func() { + cancel() + r.wg.Wait() + store.Close() + }) + return ctx, r, items[0].ID +} + +// waitEntered fails fast instead of hanging when a stage never starts. +func waitEntered(t *testing.T, entered <-chan struct{}) { + t.Helper() + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("stage execution did not start") + } +} + +// blockingExecute is the r.execute seam for these tests: it signals each entry +// and then blocks until release is closed, so a test can hold several real runs +// executing at once and observe them through getRun/listRuns. +func blockingExecute(entered chan<- struct{}, release <-chan struct{}) func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { + entered <- struct{}{} + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } +} + +// TestDifferentWorkspacesRunConcurrently proves two runs in two different +// resolved workspaces are admitted at the same time and both report executing. +func TestDifferentWorkspacesRunConcurrently(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + a, b := t.TempDir(), t.TempDir() + entered := make(chan struct{}, 2) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatalf("second workspace was blocked by the first: %v", err) + } + waitEntered(t, entered) + for _, id := range []int64{first.ID, second.ID} { + detail, err := r.getRun(id) + if err != nil || !detail.Executing { + t.Fatalf("run %d was not reported executing: %v", id, err) + } + } + runs, err := r.listRuns() + if err != nil { + t.Fatal(err) + } + executing := map[int64]bool{} + for _, run := range runs { + executing[run.ID] = run.Executing + } + if !executing[first.ID] || !executing[second.ID] { + t.Fatalf("listRuns did not report both runs executing: %+v", runs) + } + close(release) + r.wg.Wait() + for _, id := range []int64{first.ID, second.ID} { + detail, err := r.getRun(id) + if err != nil || detail.Status != "succeeded" || detail.Executing { + t.Fatalf("run %d did not reach succeeded: %+v %v", id, detail, err) + } + } +} + +// TestSameWorkspaceSecondRunIsRefused proves a different request key in the same +// resolved workspace is refused, naming the conflicting run and the remedy. +func TestSameWorkspaceSecondRunIsRefused(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + workspace := t.TempDir() + entered := make(chan struct{}, 2) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "first", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "second", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) || !strings.Contains(err.Error(), "different workspace") { + t.Fatalf("same-workspace run was not refused with an actionable conflict: %v", err) + } + close(release) + r.wg.Wait() +} + +// TestConcurrentSameWorkspaceStartsRefuseAllButOne proves the check and the +// registration of r.active are atomic under r.mu: two simultaneous starts for +// one workspace cannot both succeed. +func TestConcurrentSameWorkspaceStartsRefuseAllButOne(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + workspace := t.TempDir() + entered := make(chan struct{}, 2) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + results := make(chan error, 2) + var wg sync.WaitGroup + for _, key := range []string{"one", "two"} { + wg.Add(1) + go func(key string) { + defer wg.Done() + _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: key, Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + results <- err + }(key) + } + wg.Wait() + close(results) + refused := 0 + for err := range results { + if err != nil { + refused++ + } + } + if refused != 1 { + t.Fatalf("expected exactly one refusal for one workspace, got %d", refused) + } + waitEntered(t, entered) + close(release) + r.wg.Wait() +} + +// TestSymlinkedWorkspaceConflictsWithItsTarget proves two spellings of one +// directory (symlink vs real path) resolve equal and conflict. +func TestSymlinkedWorkspaceConflictsWithItsTarget(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + root := t.TempDir() + real := filepath.Join(root, "repo") + if err := os.Mkdir(real, 0700); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "linked-repo") + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + entered := make(chan struct{}, 2) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "real", Workspace: real, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "link", Workspace: link, Input: map[string]any{"prompt": "task"}}) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) { + t.Fatalf("symlink and target did not resolve to the same workspace: %v", err) + } + close(release) + r.wg.Wait() +} + +// TestDefaultAndExplicitWorkspaceConflict proves the default cwd and an +// explicit absolute spelling of the same directory resolve equal and conflict. +func TestDefaultAndExplicitWorkspaceConflict(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + entered := make(chan struct{}, 2) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: cwd, Input: map[string]any{"prompt": "task"}}) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) { + t.Fatalf("default cwd and its explicit absolute path did not conflict: %v", err) + } + close(release) + r.wg.Wait() +} + +// TestIdempotentRepeatWhileActive proves repeating one request key returns the +// same active run whether the run is in an explicit workspace or the default. +func TestIdempotentRepeatWhileActive(t *testing.T) { + _, r, pipelineID := concurrentStore(t) + entered := make(chan struct{}, 4) + release := make(chan struct{}) + r.execute = blockingExecute(entered, release) + explicit := t.TempDir() + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + repeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}}) + if err != nil || repeat.ID != first.ID || !repeat.Executing { + t.Fatalf("explicit-workspace repeat was not idempotent: %+v %v", repeat, err) + } + // Default workspace: no Workspace field, resolved from the process cwd. + defaultFirst, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + defaultRepeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}) + if err != nil || defaultRepeat.ID != defaultFirst.ID || !defaultRepeat.Executing { + t.Fatalf("default-workspace repeat was not idempotent: %+v %v", defaultRepeat, err) + } + close(release) + r.wg.Wait() +} + +// TestRestartRecoveryStillRefuses proves the existing guard is unchanged: an +// agent session left active with no in-memory active run blocks new work. +func TestRestartRecoveryStillRefuses(t *testing.T) { + ctx, r, pipelineID := concurrentStore(t) + d, err := r.store.Get(ctx, pipelineID, 1) + if err != nil { + t.Fatal(err) + } + workspace := t.TempDir() + id, _, err := r.store.CreateRun(ctx, d, "leftover", workspace, false, nil, map[string]any{"prompt": "task"}) + if err != nil { + t.Fatal(err) + } + if err := r.store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "task"}); err != nil { + t.Fatal(err) + } + if err := r.store.RunSession(ctx, id, 1, "ses_leftover", true, "v2", ""); err != nil { + t.Fatal(err) + } + if len(r.active) != 0 { + t.Fatal("test expected no in-memory active runs") + } + _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "new", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + if err == nil || !strings.Contains(err.Error(), "restart recovery must interrupt it") { + t.Fatalf("restart-recovery refusal was not preserved: %v", err) + } +} + +// TestRetryConflictsOnlyWithSameWorkspaceActiveRun proves retry forwards the +// original run's workspace and is refused only by an active run in that same +// workspace. +func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) { + ctx, r, pipelineID := concurrentStore(t) + a, b := t.TempDir(), t.TempDir() + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{}, fmt.Errorf("deliberate failure") + } + original, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "original", Workspace: a, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + failed, err := r.store.GetRun(ctx, original.ID) + if err != nil || failed.Status != "failed" { + t.Fatalf("original run did not fail as intended: %+v %v", failed, err) + } + resolvedB, err := filepath.EvalSymlinks(b) + if err != nil { + t.Fatal(err) + } + entered := make(chan struct{}, 4) + release := make(chan struct{}) + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + entered <- struct{}{} + if input["pwd"] == resolvedB { + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + activeA, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-a", Workspace: a, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + if _, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-a"}); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", activeA.ID)) || !strings.Contains(err.Error(), "different workspace") { + t.Fatalf("retry did not conflict with same-workspace active run: %v", err) + } + // A different active workspace must not block the retry of the original. + if _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-b", Workspace: b, Input: map[string]any{"prompt": "task"}}); err != nil { + t.Fatal(err) + } + waitEntered(t, entered) + retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-b"}) + if err != nil { + t.Fatalf("retry was blocked by an unrelated workspace: %v", err) + } + if retried.Workspace != failed.Workspace { + t.Fatalf("retry did not reuse the original workspace: %q != %q", retried.Workspace, failed.Workspace) + } + close(release) + r.wg.Wait() +} diff --git a/internal/server/retry_cleanup_test.go b/internal/server/retry_cleanup_test.go index 75fb2ad9..ffe00230 100644 --- a/internal/server/retry_cleanup_test.go +++ b/internal/server/retry_cleanup_test.go @@ -34,7 +34,7 @@ func TestRetryWaitsForExecutionCleanupEvenAfterTerminalStatus(t *testing.T) { t.Fatal(err) } // Simulate the interval between persisting terminal status and final cleanup. - r.active[id] = func() {} + r.active[id] = activeRun{cancel: func() {}} detail, err := r.getRun(id) if err != nil || !detail.Executing { t.Fatal("terminal run not marked executing during cleanup", detail, err) diff --git a/internal/server/runner.go b/internal/server/runner.go index 10c7c2b5..abb2f992 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -19,11 +19,19 @@ import ( "git.a-wan.net/alan/pipelines/internal/stage" ) +// activeRun records one in-flight run's cancellation seam and the resolved +// workspace it is changing. A single map is the only active-run state, so the +// workspace cannot drift from the cancel function. +type activeRun struct { + cancel context.CancelFunc + workspace string +} + type runner struct { ctx context.Context store *pipeline.Store mu sync.Mutex - active map[int64]context.CancelFunc + active map[int64]activeRun execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error) wg sync.WaitGroup schedulerWG sync.WaitGroup @@ -50,7 +58,7 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { if err := s.RecoverRuns(ctx); err != nil { return nil, err } - return &runner{ctx: ctx, store: s, active: map[int64]context.CancelFunc{}}, nil + return &runner{ctx: ctx, store: s, active: map[int64]activeRun{}}, nil } func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) { @@ -166,7 +174,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru return nil, err } if existing == 0 { - return nil, fmt.Errorf("another run is active; inspect it before starting more work") + // The workspace is the only scope a run reserves: refuse a second + // run only when an active run is already changing this exact + // resolved directory. One active run per workspace keeps this scan + // finding at most one match, so the message is deterministic. + for id, entry := range r.active { + if entry.workspace == workspace { + return nil, fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace) + } + } } } id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, retryOf...) @@ -175,7 +191,7 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru } if created { ctx, cancel := context.WithCancel(r.ctx) - r.active[id] = cancel + r.active[id] = activeRun{cancel: cancel, workspace: workspace} r.wg.Add(1) go func() { defer r.wg.Done() @@ -247,11 +263,11 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { func (r *runner) cancel(id int64) error { r.mu.Lock() defer r.mu.Unlock() - cancel, ok := r.active[id] + entry, ok := r.active[id] if !ok { return fmt.Errorf("run is not active") } - cancel() + entry.cancel() return nil } diff --git a/internal/server/runner_test.go b/internal/server/runner_test.go index f9ba97e6..2da8d9fe 100644 --- a/internal/server/runner_test.go +++ b/internal/server/runner_test.go @@ -3,6 +3,7 @@ package server import ( "context" "errors" + "fmt" "path/filepath" "strings" "testing" @@ -56,8 +57,8 @@ func TestRunnerRoutingPersistenceAndIdempotency(t *testing.T) { } other := request other.RequestKey = "other" - if _, err := r.start(other); err == nil { - t.Fatal("overlapping execution accepted") + if _, err := r.start(other); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", run.ID)) || !strings.Contains(err.Error(), "different workspace") { + t.Fatal("overlapping execution was not refused with the conflicting run", err) } close(release) r.wg.Wait()