package server import ( "context" "fmt" "os" "path/filepath" "strings" "sync" "testing" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) func concurrentStore(t *testing.T) (context.Context, *runner, int64) { t.Helper() return newConcurrentRunner(t, pipeline.Template([]string{"one"}, true)) } // newConcurrentRunner builds the shared test store/runner around a submitted // pipeline source. Tests that need an agent stage (useAgent true) must pass a // source with a non-empty tools block: the default template's empty tools block // explicitly denies tools, so useAgent is false and the step-completion // quiescence check is never exercised. func newConcurrentRunner(t *testing.T, source string) (context.Context, *runner, int64) { t.Helper() ctx, cancel := context.WithCancel(context.Background()) store, err := pipeline.Open(filepath.Join(t.TempDir(), "concurrent.db")) if err != nil { cancel() t.Fatal(err) } if _, err := store.Submit(ctx, "test", source); err != nil { cancel() store.Close() t.Fatal(err) } items, err := store.List(ctx) if err != nil { cancel() store.Close() t.Fatal(err) } r, err := newRunner(ctx, store) if err != nil { cancel() store.Close() t.Fatal(err) } t.Cleanup(func() { cancel() r.wg.Wait() store.Close() }) return ctx, r, items[0].ID } // agentStore returns a runner whose pipeline has an agent stage (a non-empty, // explicitly declared tools block), so the step-completion quiescence check runs. func agentStore(t *testing.T) (context.Context, *runner, int64) { t.Helper() source := strings.Replace(pipeline.Template([]string{"one"}, true), "```tools\n```", "```tools\nread\nshell\n```", 1) return newConcurrentRunner(t, source) } // waitEntered fails fast instead of hanging when a stage never starts. func waitEntered(t *testing.T, entered <-chan struct{}) { t.Helper() select { case <-entered: case <-time.After(10 * time.Second): t.Fatal("stage execution did not start") } } // blockingExecute is the r.execute seam for these tests: it signals each entry // and then blocks until release is closed, so a test can hold several real runs // executing at once and observe them through getRun/listRuns. func blockingExecute(entered chan<- struct{}, release <-chan struct{}) func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { entered <- struct{}{} select { case <-release: case <-ctx.Done(): return stage.Result{}, ctx.Err() } return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } } // TestDifferentWorkspacesRunConcurrently proves two runs in two different // resolved workspaces are admitted at the same time and both report executing. func TestDifferentWorkspacesRunConcurrently(t *testing.T) { _, r, pipelineID := concurrentStore(t) a, b := t.TempDir(), t.TempDir() entered := make(chan struct{}, 2) release := make(chan struct{}) r.execute = blockingExecute(entered, release) first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } waitEntered(t, entered) second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatalf("second workspace was blocked by the first: %v", err) } waitEntered(t, entered) for _, id := range []int64{first.ID, second.ID} { detail, err := r.getRun(id) if err != nil || !detail.Executing { t.Fatalf("run %d was not reported executing: %v", id, err) } } runs, err := r.listRuns() if err != nil { t.Fatal(err) } executing := map[int64]bool{} for _, run := range runs { executing[run.ID] = run.Executing } if !executing[first.ID] || !executing[second.ID] { t.Fatalf("listRuns did not report both runs executing: %+v", runs) } close(release) r.wg.Wait() for _, id := range []int64{first.ID, second.ID} { detail, err := r.getRun(id) if err != nil || detail.Status != "succeeded" || detail.Executing { t.Fatalf("run %d did not reach succeeded: %+v %v", id, detail, err) } } } // TestSameWorkspaceSecondRunIsRefused proves a different request key in the same // resolved workspace is refused, naming the conflicting run and the remedy. func TestSameWorkspaceSecondRunIsRefused(t *testing.T) { _, r, pipelineID := concurrentStore(t) workspace := t.TempDir() entered := make(chan struct{}, 2) release := make(chan struct{}) r.execute = blockingExecute(entered, release) first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "first", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } waitEntered(t, entered) _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "second", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) || !strings.Contains(err.Error(), "different workspace") { t.Fatalf("same-workspace run was not refused with an actionable conflict: %v", err) } close(release) r.wg.Wait() } // TestConcurrentSameWorkspaceStartsRefuseAllButOne proves the check and the // registration of r.active are atomic under r.mu: two simultaneous starts for // one workspace cannot both succeed. func TestConcurrentSameWorkspaceStartsRefuseAllButOne(t *testing.T) { _, r, pipelineID := concurrentStore(t) workspace := t.TempDir() entered := make(chan struct{}, 2) release := make(chan struct{}) r.execute = blockingExecute(entered, release) results := make(chan error, 2) var wg sync.WaitGroup for _, key := range []string{"one", "two"} { wg.Add(1) go func(key string) { defer wg.Done() _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: key, Workspace: workspace, Input: map[string]any{"prompt": "task"}}) results <- err }(key) } wg.Wait() close(results) refused := 0 for err := range results { if err != nil { refused++ } } if refused != 1 { t.Fatalf("expected exactly one refusal for one workspace, got %d", refused) } waitEntered(t, entered) close(release) r.wg.Wait() } // TestSymlinkedWorkspaceConflictsWithItsTarget proves two spellings of one // directory (symlink vs real path) resolve equal and conflict. func TestSymlinkedWorkspaceConflictsWithItsTarget(t *testing.T) { _, r, pipelineID := concurrentStore(t) root := t.TempDir() real := filepath.Join(root, "repo") if err := os.Mkdir(real, 0700); err != nil { t.Fatal(err) } link := filepath.Join(root, "linked-repo") if err := os.Symlink(real, link); err != nil { t.Fatal(err) } entered := make(chan struct{}, 2) release := make(chan struct{}) r.execute = blockingExecute(entered, release) first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "real", Workspace: real, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } waitEntered(t, entered) _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "link", Workspace: link, Input: map[string]any{"prompt": "task"}}) if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) { t.Fatalf("symlink and target did not resolve to the same workspace: %v", err) } close(release) r.wg.Wait() } // TestDefaultAndExplicitWorkspaceConflict proves the default cwd and an // explicit absolute spelling of the same directory resolve equal and conflict. func TestDefaultAndExplicitWorkspaceConflict(t *testing.T) { _, r, pipelineID := concurrentStore(t) cwd, err := os.Getwd() if err != nil { t.Fatal(err) } entered := make(chan struct{}, 2) release := make(chan struct{}) r.execute = blockingExecute(entered, release) first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } waitEntered(t, entered) _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: cwd, Input: map[string]any{"prompt": "task"}}) if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) { t.Fatalf("default cwd and its explicit absolute path did not conflict: %v", err) } close(release) r.wg.Wait() } // TestIdempotentRepeatWhileActive proves repeating one request key returns the // same active run whether the run is in an explicit workspace or the default. func TestIdempotentRepeatWhileActive(t *testing.T) { _, r, pipelineID := concurrentStore(t) entered := make(chan struct{}, 4) release := make(chan struct{}) r.execute = blockingExecute(entered, release) explicit := t.TempDir() first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } waitEntered(t, entered) repeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}}) if err != nil || repeat.ID != first.ID || !repeat.Executing { t.Fatalf("explicit-workspace repeat was not idempotent: %+v %v", repeat, err) } // Default workspace: no Workspace field, resolved from the process cwd. defaultFirst, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } waitEntered(t, entered) defaultRepeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}) if err != nil || defaultRepeat.ID != defaultFirst.ID || !defaultRepeat.Executing { t.Fatalf("default-workspace repeat was not idempotent: %+v %v", defaultRepeat, err) } close(release) r.wg.Wait() } // TestRestartRecoveryStillRefuses proves the existing guard is unchanged: an // agent session left active with no in-memory active run blocks new work. func TestRestartRecoveryStillRefuses(t *testing.T) { ctx, r, pipelineID := concurrentStore(t) d, err := r.store.Get(ctx, pipelineID, 1) if err != nil { t.Fatal(err) } workspace := t.TempDir() id, _, err := r.store.CreateRun(ctx, d, "leftover", workspace, false, nil, map[string]any{"prompt": "task"}) if err != nil { t.Fatal(err) } if err := r.store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "task"}); err != nil { t.Fatal(err) } if err := r.store.RunSession(ctx, id, 1, "ses_leftover", true, "v2", ""); err != nil { t.Fatal(err) } if len(r.active) != 0 { t.Fatal("test expected no in-memory active runs") } _, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "new", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) if err == nil || !strings.Contains(err.Error(), "restart recovery must interrupt it") { t.Fatalf("restart-recovery refusal was not preserved: %v", err) } } // TestRetryConflictsOnlyWithSameWorkspaceActiveRun proves retry forwards the // original run's workspace and is refused only by an active run in that same // workspace. func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) { ctx, r, pipelineID := concurrentStore(t) a, b := t.TempDir(), t.TempDir() r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{}, fmt.Errorf("deliberate failure") } original, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "original", Workspace: a, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } r.wg.Wait() failed, err := r.store.GetRun(ctx, original.ID) if err != nil || failed.Status != "failed" { t.Fatalf("original run did not fail as intended: %+v %v", failed, err) } resolvedB, err := filepath.EvalSymlinks(b) if err != nil { t.Fatal(err) } entered := make(chan struct{}, 4) release := make(chan struct{}) r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { entered <- struct{}{} if input["pwd"] == resolvedB { select { case <-release: case <-ctx.Done(): return stage.Result{}, ctx.Err() } } return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } activeA, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-a", Workspace: a, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } waitEntered(t, entered) if _, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-a"}); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", activeA.ID)) || !strings.Contains(err.Error(), "different workspace") { t.Fatalf("retry did not conflict with same-workspace active run: %v", err) } // A different active workspace must not block the retry of the original. if _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-b", Workspace: b, Input: map[string]any{"prompt": "task"}}); err != nil { t.Fatal(err) } waitEntered(t, entered) retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-b"}) if err != nil { t.Fatalf("retry was blocked by an unrelated workspace: %v", err) } if retried.Workspace != failed.Workspace { t.Fatalf("retry did not reuse the original workspace: %q != %q", retried.Workspace, failed.Workspace) } close(release) r.wg.Wait() } // waitTerminal polls one run until it leaves queued/running, failing fast instead // of hanging. It is scoped to the run so a blocked sibling does not stall it. func waitTerminal(t *testing.T, r *runner, id int64) *protocol.RunDetail { t.Helper() deadline := time.Now().Add(10 * time.Second) for { detail, err := r.getRun(id) if err != nil { t.Fatal(err) } switch detail.Status { case "succeeded", "failed", "canceled": return detail } if time.Now().After(deadline) { t.Fatalf("run %d did not reach a terminal state: %+v", id, detail) } time.Sleep(5 * time.Millisecond) } } // TestOverlappingRunsDoNotFailQuiescence proves the step-completion check is // scoped to the run: with two agent runs in different workspaces each recording // its OWN session active, releasing run A lets A reach succeeded while run B's // session is still active, and vice versa. A server-wide check would fail A // deterministically against B's active session. func TestOverlappingRunsDoNotFailQuiescence(t *testing.T) { _, r, pipelineID := agentStore(t) a, b := t.TempDir(), t.TempDir() type agentRun struct { id int64 session string } var mu sync.Mutex byWorkspace := map[string]agentRun{} releases := map[string]chan struct{}{} ready := make(chan struct{}) entered := make(chan string, 2) resolve := func(dir string) string { t.Helper() resolved, err := filepath.EvalSymlinks(dir) if err != nil { t.Fatal(err) } return resolved } r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { <-ready workspace, _ := input["pwd"].(string) mu.Lock() agent, ok := byWorkspace[workspace] release := releases[workspace] mu.Unlock() if !ok { return stage.Result{}, fmt.Errorf("execute for unknown workspace %q", workspace) } if err := r.store.RunSession(context.Background(), agent.id, 1, agent.session, true, "v2", ""); err != nil { return stage.Result{}, err } entered <- workspace select { case <-release: case <-ctx.Done(): return stage.Result{}, ctx.Err() } if err := r.store.RunSession(context.Background(), agent.id, 1, agent.session, false, "v2", ""); err != nil { return stage.Result{}, err } return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatalf("second workspace was blocked by the first: %v", err) } if first.Workspace != resolve(a) || second.Workspace != resolve(b) { t.Fatalf("unexpected resolved workspaces: %q, %q", first.Workspace, second.Workspace) } mu.Lock() byWorkspace[first.Workspace] = agentRun{id: first.ID, session: "ses_a"} byWorkspace[second.Workspace] = agentRun{id: second.ID, session: "ses_b"} releases[first.Workspace] = make(chan struct{}) releases[second.Workspace] = make(chan struct{}) mu.Unlock() close(ready) waitEnteredString(t, entered) waitEnteredString(t, entered) for _, id := range []int64{first.ID, second.ID} { detail, err := r.getRun(id) if err != nil || !detail.Executing { t.Fatalf("run %d was not reported executing: %v", id, err) } } // Release A only; B's own session stays active while A finishes. close(releases[first.Workspace]) done := waitTerminal(t, r, first.ID) if done.Status != "succeeded" || strings.Contains(done.Diagnostic, "agent quiescence not confirmed") { t.Fatalf("run A failed despite its own session being idle: %+v", done) } if detail, err := r.getRun(second.ID); err != nil || !detail.Executing { t.Fatalf("run B stopped while A finished: %+v %v", detail, err) } close(releases[second.Workspace]) done = waitTerminal(t, r, second.ID) if done.Status != "succeeded" || strings.Contains(done.Diagnostic, "agent quiescence not confirmed") { t.Fatalf("run B failed despite its own session being idle: %+v", done) } r.wg.Wait() } // waitEnteredString fails fast instead of hanging when a stage never starts; it // carries the entering workspace for tests that key on it. func waitEnteredString(t *testing.T, entered <-chan string) { t.Helper() select { case <-entered: case <-time.After(10 * time.Second): t.Fatal("stage execution did not start") } } // TestOwnActiveSessionStillFailsRun proves a run still fails when one of its OWN // agent sessions is left active after the stage completes. func TestOwnActiveSessionStillFailsRun(t *testing.T) { _, r, pipelineID := agentStore(t) workspace := t.TempDir() ready := make(chan struct{}) var runID int64 r.execute = func(_ context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { <-ready if err := r.store.RunSession(context.Background(), runID, 1, "ses_own", true, "v2", ""); err != nil { return stage.Result{}, err } return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "own", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } runID = run.ID close(ready) r.wg.Wait() detail, err := r.store.GetRun(context.Background(), run.ID) if err != nil { t.Fatal(err) } if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "agent quiescence not confirmed") { t.Fatalf("own active session did not fail the run: %+v", detail) } }