Allow concurrent runs in different workspaces

Track each active run's resolved workspace alongside its cancel function
in a single runner.active struct map. A new request key is now refused
only when an already-active run is changing the same resolved workspace;
the refusal names the conflicting run id and advises waiting or using a
different workspace. Workspace resolution is unchanged (default cwd,
absolute request workspace, EvalSymlinks), so two spellings of one
directory still compare equal. The conflict check and registration stay
inside the existing r.mu critical section, so concurrent starts for one
workspace cannot both pass. Idempotent repeats of an active request key
and the restart-recovery guard are unchanged.

Add concurrency_test.go covering concurrent different-workspace runs,
same-workspace refusal, symlink/default-cwd equivalence, idempotent
repeats, restart recovery and retry workspace scoping; strengthen the
runner_test.go refusal assertion. Update docs/cli.md,
docs/first-version-testing.md and the packaged operations skill reference
to describe per-workspace concurrency.

Assisted-by: DeepSeek V4.1 Flash
This commit is contained in:
2026-10-15 15:08:44 +01:00
parent 1828171a9c
commit 2618f0d244
7 changed files with 390 additions and 20 deletions
+6 -3
View File
@@ -100,9 +100,12 @@ Every submission, including agent-issued `submit`/`rpc`, still uses server valid
## Drafts in the TUI
The server admits one active run at a time. A new request key is rejected while
another run is active; inspect that run and wait for it to finish. Reusing its
original request key reattaches idempotently.
The server runs one active run per resolved workspace. A new request key in a
different workspace is admitted immediately and executes concurrently; a second
run in a workspace an active run is already changing is refused, naming the
conflicting run and advising you to wait or use a different workspace (for
example another checkout of the repository). Reusing the original request key
reattaches idempotently.
Use `:templates` for five simple, manual-only starters: task, summary, review,
research, and plan/implement/review. **Submit template** (`s` or Enter) asks for a
+3 -1
View File
@@ -95,7 +95,9 @@ delivery to your phone. A summary starter does not send alerts automatically.
## Known boundaries
- Sequential stages only; no parallel execution or built-in human-approval stages.
- Sequential stages within a run; no parallel branches or built-in
human-approval stages. Independent runs in different workspaces do execute
concurrently.
- Cancellation/retry is **not rollback**. Retry starts from the entry stage with
fresh sessions and the pinned definition; earlier side effects can repeat.
- Diffs are bounded stage-start/stage-end observations, not full backups or proof
@@ -21,13 +21,15 @@ parallel/distributed task engine or a guarantee that model reports are correct.
client does not upload its repository or make host credentials available remotely.
- Host and Docker OpenCode credentials are separate. Preserve setup-selected model
inheritance; do not select a provider just because its ID appears in an example.
- Runtime caps are 30 stage steps and five visits to a stage. There are no parallel
runs on one server: admission rejects new work while another run is active;
reusing that active run's request key reattaches idempotently. Inspect and wait
for the existing run rather than starting duplicates. There are no parallel
branches or built-in human approval stages. Optional worktrees, full backup/restore,
automatic database sync and independent verifier stage types are future ideas,
not existing capabilities. Do not invent them in definitions.
- Runtime caps are 30 stage steps and five visits to a stage. Runs in different
resolved workspaces execute concurrently; only a second run in a workspace an
active run is already changing is refused, naming the conflicting run and
advising you to wait or use a different workspace (for example another
checkout). Reusing that active run's request key reattaches idempotently.
There are no parallel branches or built-in human approval stages. Optional
worktrees, full backup/restore, automatic database sync and independent
verifier stage types are future ideas, not existing capabilities. Do not
invent them in definitions.
- Services are native or Docker-based. With an existing Docker installation do not
start a competing native server. Server APIs are unauthenticated: trusted networks
only; never recommend public exposure. Tool scope is not a universal OS sandbox.
+346
View File
@@ -0,0 +1,346 @@
package server
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func concurrentStore(t *testing.T) (context.Context, *runner, int64) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
store, err := pipeline.Open(filepath.Join(t.TempDir(), "concurrent.db"))
if err != nil {
cancel()
t.Fatal(err)
}
if _, err := store.Submit(ctx, "test", pipeline.Template([]string{"one"}, true)); err != nil {
cancel()
store.Close()
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
cancel()
store.Close()
t.Fatal(err)
}
r, err := newRunner(ctx, store)
if err != nil {
cancel()
store.Close()
t.Fatal(err)
}
t.Cleanup(func() {
cancel()
r.wg.Wait()
store.Close()
})
return ctx, r, items[0].ID
}
// waitEntered fails fast instead of hanging when a stage never starts.
func waitEntered(t *testing.T, entered <-chan struct{}) {
t.Helper()
select {
case <-entered:
case <-time.After(10 * time.Second):
t.Fatal("stage execution did not start")
}
}
// blockingExecute is the r.execute seam for these tests: it signals each entry
// and then blocks until release is closed, so a test can hold several real runs
// executing at once and observe them through getRun/listRuns.
func blockingExecute(entered chan<- struct{}, release <-chan struct{}) func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
entered <- struct{}{}
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
}
// TestDifferentWorkspacesRunConcurrently proves two runs in two different
// resolved workspaces are admitted at the same time and both report executing.
func TestDifferentWorkspacesRunConcurrently(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
a, b := t.TempDir(), t.TempDir()
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatalf("second workspace was blocked by the first: %v", err)
}
waitEntered(t, entered)
for _, id := range []int64{first.ID, second.ID} {
detail, err := r.getRun(id)
if err != nil || !detail.Executing {
t.Fatalf("run %d was not reported executing: %v", id, err)
}
}
runs, err := r.listRuns()
if err != nil {
t.Fatal(err)
}
executing := map[int64]bool{}
for _, run := range runs {
executing[run.ID] = run.Executing
}
if !executing[first.ID] || !executing[second.ID] {
t.Fatalf("listRuns did not report both runs executing: %+v", runs)
}
close(release)
r.wg.Wait()
for _, id := range []int64{first.ID, second.ID} {
detail, err := r.getRun(id)
if err != nil || detail.Status != "succeeded" || detail.Executing {
t.Fatalf("run %d did not reach succeeded: %+v %v", id, detail, err)
}
}
}
// TestSameWorkspaceSecondRunIsRefused proves a different request key in the same
// resolved workspace is refused, naming the conflicting run and the remedy.
func TestSameWorkspaceSecondRunIsRefused(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
workspace := t.TempDir()
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "first", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "second", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) || !strings.Contains(err.Error(), "different workspace") {
t.Fatalf("same-workspace run was not refused with an actionable conflict: %v", err)
}
close(release)
r.wg.Wait()
}
// TestConcurrentSameWorkspaceStartsRefuseAllButOne proves the check and the
// registration of r.active are atomic under r.mu: two simultaneous starts for
// one workspace cannot both succeed.
func TestConcurrentSameWorkspaceStartsRefuseAllButOne(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
workspace := t.TempDir()
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
results := make(chan error, 2)
var wg sync.WaitGroup
for _, key := range []string{"one", "two"} {
wg.Add(1)
go func(key string) {
defer wg.Done()
_, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: key, Workspace: workspace, Input: map[string]any{"prompt": "task"}})
results <- err
}(key)
}
wg.Wait()
close(results)
refused := 0
for err := range results {
if err != nil {
refused++
}
}
if refused != 1 {
t.Fatalf("expected exactly one refusal for one workspace, got %d", refused)
}
waitEntered(t, entered)
close(release)
r.wg.Wait()
}
// TestSymlinkedWorkspaceConflictsWithItsTarget proves two spellings of one
// directory (symlink vs real path) resolve equal and conflict.
func TestSymlinkedWorkspaceConflictsWithItsTarget(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
root := t.TempDir()
real := filepath.Join(root, "repo")
if err := os.Mkdir(real, 0700); err != nil {
t.Fatal(err)
}
link := filepath.Join(root, "linked-repo")
if err := os.Symlink(real, link); err != nil {
t.Fatal(err)
}
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "real", Workspace: real, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "link", Workspace: link, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) {
t.Fatalf("symlink and target did not resolve to the same workspace: %v", err)
}
close(release)
r.wg.Wait()
}
// TestDefaultAndExplicitWorkspaceConflict proves the default cwd and an
// explicit absolute spelling of the same directory resolve equal and conflict.
func TestDefaultAndExplicitWorkspaceConflict(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
cwd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: cwd, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) {
t.Fatalf("default cwd and its explicit absolute path did not conflict: %v", err)
}
close(release)
r.wg.Wait()
}
// TestIdempotentRepeatWhileActive proves repeating one request key returns the
// same active run whether the run is in an explicit workspace or the default.
func TestIdempotentRepeatWhileActive(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
entered := make(chan struct{}, 4)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
explicit := t.TempDir()
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
repeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}})
if err != nil || repeat.ID != first.ID || !repeat.Executing {
t.Fatalf("explicit-workspace repeat was not idempotent: %+v %v", repeat, err)
}
// Default workspace: no Workspace field, resolved from the process cwd.
defaultFirst, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
defaultRepeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}})
if err != nil || defaultRepeat.ID != defaultFirst.ID || !defaultRepeat.Executing {
t.Fatalf("default-workspace repeat was not idempotent: %+v %v", defaultRepeat, err)
}
close(release)
r.wg.Wait()
}
// TestRestartRecoveryStillRefuses proves the existing guard is unchanged: an
// agent session left active with no in-memory active run blocks new work.
func TestRestartRecoveryStillRefuses(t *testing.T) {
ctx, r, pipelineID := concurrentStore(t)
d, err := r.store.Get(ctx, pipelineID, 1)
if err != nil {
t.Fatal(err)
}
workspace := t.TempDir()
id, _, err := r.store.CreateRun(ctx, d, "leftover", workspace, false, nil, map[string]any{"prompt": "task"})
if err != nil {
t.Fatal(err)
}
if err := r.store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "task"}); err != nil {
t.Fatal(err)
}
if err := r.store.RunSession(ctx, id, 1, "ses_leftover", true, "v2", ""); err != nil {
t.Fatal(err)
}
if len(r.active) != 0 {
t.Fatal("test expected no in-memory active runs")
}
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "new", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), "restart recovery must interrupt it") {
t.Fatalf("restart-recovery refusal was not preserved: %v", err)
}
}
// TestRetryConflictsOnlyWithSameWorkspaceActiveRun proves retry forwards the
// original run's workspace and is refused only by an active run in that same
// workspace.
func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) {
ctx, r, pipelineID := concurrentStore(t)
a, b := t.TempDir(), t.TempDir()
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, fmt.Errorf("deliberate failure")
}
original, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "original", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
failed, err := r.store.GetRun(ctx, original.ID)
if err != nil || failed.Status != "failed" {
t.Fatalf("original run did not fail as intended: %+v %v", failed, err)
}
resolvedB, err := filepath.EvalSymlinks(b)
if err != nil {
t.Fatal(err)
}
entered := make(chan struct{}, 4)
release := make(chan struct{})
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
entered <- struct{}{}
if input["pwd"] == resolvedB {
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
activeA, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-a", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
if _, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-a"}); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", activeA.ID)) || !strings.Contains(err.Error(), "different workspace") {
t.Fatalf("retry did not conflict with same-workspace active run: %v", err)
}
// A different active workspace must not block the retry of the original.
if _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-b", Workspace: b, Input: map[string]any{"prompt": "task"}}); err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-b"})
if err != nil {
t.Fatalf("retry was blocked by an unrelated workspace: %v", err)
}
if retried.Workspace != failed.Workspace {
t.Fatalf("retry did not reuse the original workspace: %q != %q", retried.Workspace, failed.Workspace)
}
close(release)
r.wg.Wait()
}
+1 -1
View File
@@ -34,7 +34,7 @@ func TestRetryWaitsForExecutionCleanupEvenAfterTerminalStatus(t *testing.T) {
t.Fatal(err)
}
// Simulate the interval between persisting terminal status and final cleanup.
r.active[id] = func() {}
r.active[id] = activeRun{cancel: func() {}}
detail, err := r.getRun(id)
if err != nil || !detail.Executing {
t.Fatal("terminal run not marked executing during cleanup", detail, err)
+22 -6
View File
@@ -19,11 +19,19 @@ import (
"git.a-wan.net/alan/pipelines/internal/stage"
)
// activeRun records one in-flight run's cancellation seam and the resolved
// workspace it is changing. A single map is the only active-run state, so the
// workspace cannot drift from the cancel function.
type activeRun struct {
cancel context.CancelFunc
workspace string
}
type runner struct {
ctx context.Context
store *pipeline.Store
mu sync.Mutex
active map[int64]context.CancelFunc
active map[int64]activeRun
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
wg sync.WaitGroup
schedulerWG sync.WaitGroup
@@ -50,7 +58,7 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
if err := s.RecoverRuns(ctx); err != nil {
return nil, err
}
return &runner{ctx: ctx, store: s, active: map[int64]context.CancelFunc{}}, nil
return &runner{ctx: ctx, store: s, active: map[int64]activeRun{}}, nil
}
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
@@ -166,7 +174,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
return nil, err
}
if existing == 0 {
return nil, fmt.Errorf("another run is active; inspect it before starting more work")
// The workspace is the only scope a run reserves: refuse a second
// run only when an active run is already changing this exact
// resolved directory. One active run per workspace keeps this scan
// finding at most one match, so the message is deterministic.
for id, entry := range r.active {
if entry.workspace == workspace {
return nil, fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
}
}
}
}
id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, retryOf...)
@@ -175,7 +191,7 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
}
if created {
ctx, cancel := context.WithCancel(r.ctx)
r.active[id] = cancel
r.active[id] = activeRun{cancel: cancel, workspace: workspace}
r.wg.Add(1)
go func() {
defer r.wg.Done()
@@ -247,11 +263,11 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
func (r *runner) cancel(id int64) error {
r.mu.Lock()
defer r.mu.Unlock()
cancel, ok := r.active[id]
entry, ok := r.active[id]
if !ok {
return fmt.Errorf("run is not active")
}
cancel()
entry.cancel()
return nil
}
+3 -2
View File
@@ -3,6 +3,7 @@ package server
import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"testing"
@@ -56,8 +57,8 @@ func TestRunnerRoutingPersistenceAndIdempotency(t *testing.T) {
}
other := request
other.RequestKey = "other"
if _, err := r.start(other); err == nil {
t.Fatal("overlapping execution accepted")
if _, err := r.start(other); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", run.ID)) || !strings.Contains(err.Error(), "different workspace") {
t.Fatal("overlapping execution was not refused with the conflicting run", err)
}
close(release)
r.wg.Wait()