Files
pipelines/internal/server/concurrency_test.go
T
alan e6d00d4ee5 Scope agent session bookkeeping to the run for concurrent runs
The per-workspace concurrency added in 2618f0d exposed three places that assumed a
single run, and the second concurrent run failed with 'agent quiescence not
confirmed' because of them.

- ActiveAgentSessionsForRun(runID) scopes the end-of-step quiescence check to the
  run's own sessions, so another run's legitimately active agent no longer fails
  this one. A run still fails when its own session cannot be confirmed idle.
- The startup guard now uses ActiveSessionOwners (which carries the owning run id)
  and refuses only an ORPHANED session - one whose run is not active now - which is
  what restart recovery is actually about. A session owned by a currently active
  run no longer blocks new work.
- sessionActive examines only the target session: foreign entries in the
  /api/session/active map are ignored, the target's absence is inactive, and the
  target present with an unsupported state still fails closed.

Verified: go build, go vet and go test ./... pass, including ten concurrency tests
(two workspaces run concurrently, same-workspace refusal, symlink and default-cwd
equivalence, idempotent repeats, restart recovery, retry scoping, overlapping runs
not failing quiescence, and a run's own active session still failing it).

Operator verification, which a stage cannot perform because restarting the server
would abort the run doing it: the service was rebuilt and restarted on this
commit, then two runs in separate workspaces (/tmp/ws-c and /tmp/ws-d) executed
concurrently - both reported executing at the same time, a third run in ws-c was
refused naming run 46, and both finished succeeded with no diagnostic.
2026-10-15 15:38:50 +01:00

520 lines
19 KiB
Go

package server
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func concurrentStore(t *testing.T) (context.Context, *runner, int64) {
t.Helper()
return newConcurrentRunner(t, pipeline.Template([]string{"one"}, true))
}
// newConcurrentRunner builds the shared test store/runner around a submitted
// pipeline source. Tests that need an agent stage (useAgent true) must pass a
// source with a non-empty tools block: the default template's empty tools block
// explicitly denies tools, so useAgent is false and the step-completion
// quiescence check is never exercised.
func newConcurrentRunner(t *testing.T, source string) (context.Context, *runner, int64) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
store, err := pipeline.Open(filepath.Join(t.TempDir(), "concurrent.db"))
if err != nil {
cancel()
t.Fatal(err)
}
if _, err := store.Submit(ctx, "test", source); err != nil {
cancel()
store.Close()
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
cancel()
store.Close()
t.Fatal(err)
}
r, err := newRunner(ctx, store)
if err != nil {
cancel()
store.Close()
t.Fatal(err)
}
t.Cleanup(func() {
cancel()
r.wg.Wait()
store.Close()
})
return ctx, r, items[0].ID
}
// agentStore returns a runner whose pipeline has an agent stage (a non-empty,
// explicitly declared tools block), so the step-completion quiescence check runs.
func agentStore(t *testing.T) (context.Context, *runner, int64) {
t.Helper()
source := strings.Replace(pipeline.Template([]string{"one"}, true), "```tools\n```", "```tools\nread\nshell\n```", 1)
return newConcurrentRunner(t, source)
}
// waitEntered fails fast instead of hanging when a stage never starts.
func waitEntered(t *testing.T, entered <-chan struct{}) {
t.Helper()
select {
case <-entered:
case <-time.After(10 * time.Second):
t.Fatal("stage execution did not start")
}
}
// blockingExecute is the r.execute seam for these tests: it signals each entry
// and then blocks until release is closed, so a test can hold several real runs
// executing at once and observe them through getRun/listRuns.
func blockingExecute(entered chan<- struct{}, release <-chan struct{}) func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
entered <- struct{}{}
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
}
// TestDifferentWorkspacesRunConcurrently proves two runs in two different
// resolved workspaces are admitted at the same time and both report executing.
func TestDifferentWorkspacesRunConcurrently(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
a, b := t.TempDir(), t.TempDir()
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatalf("second workspace was blocked by the first: %v", err)
}
waitEntered(t, entered)
for _, id := range []int64{first.ID, second.ID} {
detail, err := r.getRun(id)
if err != nil || !detail.Executing {
t.Fatalf("run %d was not reported executing: %v", id, err)
}
}
runs, err := r.listRuns()
if err != nil {
t.Fatal(err)
}
executing := map[int64]bool{}
for _, run := range runs {
executing[run.ID] = run.Executing
}
if !executing[first.ID] || !executing[second.ID] {
t.Fatalf("listRuns did not report both runs executing: %+v", runs)
}
close(release)
r.wg.Wait()
for _, id := range []int64{first.ID, second.ID} {
detail, err := r.getRun(id)
if err != nil || detail.Status != "succeeded" || detail.Executing {
t.Fatalf("run %d did not reach succeeded: %+v %v", id, detail, err)
}
}
}
// TestSameWorkspaceSecondRunIsRefused proves a different request key in the same
// resolved workspace is refused, naming the conflicting run and the remedy.
func TestSameWorkspaceSecondRunIsRefused(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
workspace := t.TempDir()
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "first", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "second", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) || !strings.Contains(err.Error(), "different workspace") {
t.Fatalf("same-workspace run was not refused with an actionable conflict: %v", err)
}
close(release)
r.wg.Wait()
}
// TestConcurrentSameWorkspaceStartsRefuseAllButOne proves the check and the
// registration of r.active are atomic under r.mu: two simultaneous starts for
// one workspace cannot both succeed.
func TestConcurrentSameWorkspaceStartsRefuseAllButOne(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
workspace := t.TempDir()
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
results := make(chan error, 2)
var wg sync.WaitGroup
for _, key := range []string{"one", "two"} {
wg.Add(1)
go func(key string) {
defer wg.Done()
_, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: key, Workspace: workspace, Input: map[string]any{"prompt": "task"}})
results <- err
}(key)
}
wg.Wait()
close(results)
refused := 0
for err := range results {
if err != nil {
refused++
}
}
if refused != 1 {
t.Fatalf("expected exactly one refusal for one workspace, got %d", refused)
}
waitEntered(t, entered)
close(release)
r.wg.Wait()
}
// TestSymlinkedWorkspaceConflictsWithItsTarget proves two spellings of one
// directory (symlink vs real path) resolve equal and conflict.
func TestSymlinkedWorkspaceConflictsWithItsTarget(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
root := t.TempDir()
real := filepath.Join(root, "repo")
if err := os.Mkdir(real, 0700); err != nil {
t.Fatal(err)
}
link := filepath.Join(root, "linked-repo")
if err := os.Symlink(real, link); err != nil {
t.Fatal(err)
}
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "real", Workspace: real, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "link", Workspace: link, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) {
t.Fatalf("symlink and target did not resolve to the same workspace: %v", err)
}
close(release)
r.wg.Wait()
}
// TestDefaultAndExplicitWorkspaceConflict proves the default cwd and an
// explicit absolute spelling of the same directory resolve equal and conflict.
func TestDefaultAndExplicitWorkspaceConflict(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
cwd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: cwd, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) {
t.Fatalf("default cwd and its explicit absolute path did not conflict: %v", err)
}
close(release)
r.wg.Wait()
}
// TestIdempotentRepeatWhileActive proves repeating one request key returns the
// same active run whether the run is in an explicit workspace or the default.
func TestIdempotentRepeatWhileActive(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
entered := make(chan struct{}, 4)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
explicit := t.TempDir()
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
repeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}})
if err != nil || repeat.ID != first.ID || !repeat.Executing {
t.Fatalf("explicit-workspace repeat was not idempotent: %+v %v", repeat, err)
}
// Default workspace: no Workspace field, resolved from the process cwd.
defaultFirst, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
defaultRepeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}})
if err != nil || defaultRepeat.ID != defaultFirst.ID || !defaultRepeat.Executing {
t.Fatalf("default-workspace repeat was not idempotent: %+v %v", defaultRepeat, err)
}
close(release)
r.wg.Wait()
}
// TestRestartRecoveryStillRefuses proves the existing guard is unchanged: an
// agent session left active with no in-memory active run blocks new work.
func TestRestartRecoveryStillRefuses(t *testing.T) {
ctx, r, pipelineID := concurrentStore(t)
d, err := r.store.Get(ctx, pipelineID, 1)
if err != nil {
t.Fatal(err)
}
workspace := t.TempDir()
id, _, err := r.store.CreateRun(ctx, d, "leftover", workspace, false, nil, map[string]any{"prompt": "task"})
if err != nil {
t.Fatal(err)
}
if err := r.store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "task"}); err != nil {
t.Fatal(err)
}
if err := r.store.RunSession(ctx, id, 1, "ses_leftover", true, "v2", ""); err != nil {
t.Fatal(err)
}
if len(r.active) != 0 {
t.Fatal("test expected no in-memory active runs")
}
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "new", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), "restart recovery must interrupt it") {
t.Fatalf("restart-recovery refusal was not preserved: %v", err)
}
}
// TestRetryConflictsOnlyWithSameWorkspaceActiveRun proves retry forwards the
// original run's workspace and is refused only by an active run in that same
// workspace.
func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) {
ctx, r, pipelineID := concurrentStore(t)
a, b := t.TempDir(), t.TempDir()
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, fmt.Errorf("deliberate failure")
}
original, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "original", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
failed, err := r.store.GetRun(ctx, original.ID)
if err != nil || failed.Status != "failed" {
t.Fatalf("original run did not fail as intended: %+v %v", failed, err)
}
resolvedB, err := filepath.EvalSymlinks(b)
if err != nil {
t.Fatal(err)
}
entered := make(chan struct{}, 4)
release := make(chan struct{})
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
entered <- struct{}{}
if input["pwd"] == resolvedB {
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
activeA, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-a", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
if _, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-a"}); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", activeA.ID)) || !strings.Contains(err.Error(), "different workspace") {
t.Fatalf("retry did not conflict with same-workspace active run: %v", err)
}
// A different active workspace must not block the retry of the original.
if _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-b", Workspace: b, Input: map[string]any{"prompt": "task"}}); err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-b"})
if err != nil {
t.Fatalf("retry was blocked by an unrelated workspace: %v", err)
}
if retried.Workspace != failed.Workspace {
t.Fatalf("retry did not reuse the original workspace: %q != %q", retried.Workspace, failed.Workspace)
}
close(release)
r.wg.Wait()
}
// waitTerminal polls one run until it leaves queued/running, failing fast instead
// of hanging. It is scoped to the run so a blocked sibling does not stall it.
func waitTerminal(t *testing.T, r *runner, id int64) *protocol.RunDetail {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
detail, err := r.getRun(id)
if err != nil {
t.Fatal(err)
}
switch detail.Status {
case "succeeded", "failed", "canceled":
return detail
}
if time.Now().After(deadline) {
t.Fatalf("run %d did not reach a terminal state: %+v", id, detail)
}
time.Sleep(5 * time.Millisecond)
}
}
// TestOverlappingRunsDoNotFailQuiescence proves the step-completion check is
// scoped to the run: with two agent runs in different workspaces each recording
// its OWN session active, releasing run A lets A reach succeeded while run B's
// session is still active, and vice versa. A server-wide check would fail A
// deterministically against B's active session.
func TestOverlappingRunsDoNotFailQuiescence(t *testing.T) {
_, r, pipelineID := agentStore(t)
a, b := t.TempDir(), t.TempDir()
type agentRun struct {
id int64
session string
}
var mu sync.Mutex
byWorkspace := map[string]agentRun{}
releases := map[string]chan struct{}{}
ready := make(chan struct{})
entered := make(chan string, 2)
resolve := func(dir string) string {
t.Helper()
resolved, err := filepath.EvalSymlinks(dir)
if err != nil {
t.Fatal(err)
}
return resolved
}
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
<-ready
workspace, _ := input["pwd"].(string)
mu.Lock()
agent, ok := byWorkspace[workspace]
release := releases[workspace]
mu.Unlock()
if !ok {
return stage.Result{}, fmt.Errorf("execute for unknown workspace %q", workspace)
}
if err := r.store.RunSession(context.Background(), agent.id, 1, agent.session, true, "v2", ""); err != nil {
return stage.Result{}, err
}
entered <- workspace
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
if err := r.store.RunSession(context.Background(), agent.id, 1, agent.session, false, "v2", ""); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatalf("second workspace was blocked by the first: %v", err)
}
if first.Workspace != resolve(a) || second.Workspace != resolve(b) {
t.Fatalf("unexpected resolved workspaces: %q, %q", first.Workspace, second.Workspace)
}
mu.Lock()
byWorkspace[first.Workspace] = agentRun{id: first.ID, session: "ses_a"}
byWorkspace[second.Workspace] = agentRun{id: second.ID, session: "ses_b"}
releases[first.Workspace] = make(chan struct{})
releases[second.Workspace] = make(chan struct{})
mu.Unlock()
close(ready)
waitEnteredString(t, entered)
waitEnteredString(t, entered)
for _, id := range []int64{first.ID, second.ID} {
detail, err := r.getRun(id)
if err != nil || !detail.Executing {
t.Fatalf("run %d was not reported executing: %v", id, err)
}
}
// Release A only; B's own session stays active while A finishes.
close(releases[first.Workspace])
done := waitTerminal(t, r, first.ID)
if done.Status != "succeeded" || strings.Contains(done.Diagnostic, "agent quiescence not confirmed") {
t.Fatalf("run A failed despite its own session being idle: %+v", done)
}
if detail, err := r.getRun(second.ID); err != nil || !detail.Executing {
t.Fatalf("run B stopped while A finished: %+v %v", detail, err)
}
close(releases[second.Workspace])
done = waitTerminal(t, r, second.ID)
if done.Status != "succeeded" || strings.Contains(done.Diagnostic, "agent quiescence not confirmed") {
t.Fatalf("run B failed despite its own session being idle: %+v", done)
}
r.wg.Wait()
}
// waitEnteredString fails fast instead of hanging when a stage never starts; it
// carries the entering workspace for tests that key on it.
func waitEnteredString(t *testing.T, entered <-chan string) {
t.Helper()
select {
case <-entered:
case <-time.After(10 * time.Second):
t.Fatal("stage execution did not start")
}
}
// TestOwnActiveSessionStillFailsRun proves a run still fails when one of its OWN
// agent sessions is left active after the stage completes.
func TestOwnActiveSessionStillFailsRun(t *testing.T) {
_, r, pipelineID := agentStore(t)
workspace := t.TempDir()
ready := make(chan struct{})
var runID int64
r.execute = func(_ context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
<-ready
if err := r.store.RunSession(context.Background(), runID, 1, "ses_own", true, "v2", ""); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "own", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
runID = run.ID
close(ready)
r.wg.Wait()
detail, err := r.store.GetRun(context.Background(), run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "agent quiescence not confirmed") {
t.Fatalf("own active session did not fail the run: %+v", detail)
}
}