Files

546 lines
20 KiB
Go

package server
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func concurrentStore(t *testing.T) (context.Context, *runner, int64) {
t.Helper()
return newConcurrentRunner(t, pipeline.Template([]string{"one"}, true))
}
// submitRunner opens a fresh store, submits source through the parser and returns
// the runner, its store and the pipeline id. Callers own the returned store's
// lifetime and cleanup so tests can inspect persisted runs; newConcurrentRunner and
// the review-contract tests share this setup instead of copying it.
func submitRunner(t *testing.T, ctx context.Context, source string) (*runner, *pipeline.Store, int64) {
t.Helper()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "test.db"))
if err != nil {
t.Fatal(err)
}
if _, err := store.Submit(ctx, "test", source); err != nil {
store.Close()
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
store.Close()
t.Fatal(err)
}
r, err := newRunner(ctx, store)
if err != nil {
store.Close()
t.Fatal(err)
}
return r, store, items[0].ID
}
// startRun launches one run of a submitted pipeline and waits for it to stop,
// returning the started run detail. It is the single start helper for tests that
// only need a completed run.
func startRun(t *testing.T, r *runner, pipelineID int64, key string) *protocol.RunDetail {
t.Helper()
run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
return run
}
// newConcurrentRunner builds the shared test store/runner around a submitted
// pipeline source. Tests that need an agent stage (useAgent true) must pass a
// source with a non-empty tools block: the default template's empty tools block
// explicitly denies tools, so useAgent is false and the step-completion
// quiescence check is never exercised.
func newConcurrentRunner(t *testing.T, source string) (context.Context, *runner, int64) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
r, store, id := submitRunner(t, ctx, source)
t.Cleanup(func() {
cancel()
r.wg.Wait()
store.Close()
})
return ctx, r, id
}
// agentStore returns a runner whose pipeline has an agent stage (a non-empty,
// explicitly declared tools block), so the step-completion quiescence check runs.
func agentStore(t *testing.T) (context.Context, *runner, int64) {
t.Helper()
source := strings.Replace(pipeline.Template([]string{"one"}, true), "```tools\n```", "```tools\nread\nshell\n```", 1)
return newConcurrentRunner(t, source)
}
// waitEntered fails fast instead of hanging when a stage never starts.
func waitEntered(t *testing.T, entered <-chan struct{}) {
t.Helper()
select {
case <-entered:
case <-time.After(10 * time.Second):
t.Fatal("stage execution did not start")
}
}
// blockingExecute is the r.execute seam for these tests: it signals each entry
// and then blocks until release is closed, so a test can hold several real runs
// executing at once and observe them through getRun/listRuns.
func blockingExecute(entered chan<- struct{}, release <-chan struct{}) func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
entered <- struct{}{}
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
}
// TestDifferentWorkspacesRunConcurrently proves two runs in two different
// resolved workspaces are admitted at the same time and both report executing.
func TestDifferentWorkspacesRunConcurrently(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
a, b := t.TempDir(), t.TempDir()
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatalf("second workspace was blocked by the first: %v", err)
}
waitEntered(t, entered)
for _, id := range []int64{first.ID, second.ID} {
detail, err := r.getRun(id)
if err != nil || !detail.Executing {
t.Fatalf("run %d was not reported executing: %v", id, err)
}
}
runs, err := r.listRuns()
if err != nil {
t.Fatal(err)
}
executing := map[int64]bool{}
for _, run := range runs {
executing[run.ID] = run.Executing
}
if !executing[first.ID] || !executing[second.ID] {
t.Fatalf("listRuns did not report both runs executing: %+v", runs)
}
close(release)
r.wg.Wait()
for _, id := range []int64{first.ID, second.ID} {
detail, err := r.getRun(id)
if err != nil || detail.Status != "succeeded" || detail.Executing {
t.Fatalf("run %d did not reach succeeded: %+v %v", id, detail, err)
}
}
}
// TestSameWorkspaceSecondRunIsRefused proves a different request key in the same
// resolved workspace is refused, naming the conflicting run and the remedy.
func TestSameWorkspaceSecondRunIsRefused(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
workspace := t.TempDir()
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "first", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "second", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) || !strings.Contains(err.Error(), "different workspace") {
t.Fatalf("same-workspace run was not refused with an actionable conflict: %v", err)
}
close(release)
r.wg.Wait()
}
// TestConcurrentSameWorkspaceStartsRefuseAllButOne proves the check and the
// registration of r.active are atomic under r.mu: two simultaneous starts for
// one workspace cannot both succeed.
func TestConcurrentSameWorkspaceStartsRefuseAllButOne(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
workspace := t.TempDir()
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
results := make(chan error, 2)
var wg sync.WaitGroup
for _, key := range []string{"one", "two"} {
wg.Add(1)
go func(key string) {
defer wg.Done()
_, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: key, Workspace: workspace, Input: map[string]any{"prompt": "task"}})
results <- err
}(key)
}
wg.Wait()
close(results)
refused := 0
for err := range results {
if err != nil {
refused++
}
}
if refused != 1 {
t.Fatalf("expected exactly one refusal for one workspace, got %d", refused)
}
waitEntered(t, entered)
close(release)
r.wg.Wait()
}
// TestSymlinkedWorkspaceConflictsWithItsTarget proves two spellings of one
// directory (symlink vs real path) resolve equal and conflict.
func TestSymlinkedWorkspaceConflictsWithItsTarget(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
root := t.TempDir()
real := filepath.Join(root, "repo")
if err := os.Mkdir(real, 0700); err != nil {
t.Fatal(err)
}
link := filepath.Join(root, "linked-repo")
if err := os.Symlink(real, link); err != nil {
t.Fatal(err)
}
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "real", Workspace: real, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "link", Workspace: link, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) {
t.Fatalf("symlink and target did not resolve to the same workspace: %v", err)
}
close(release)
r.wg.Wait()
}
// TestDefaultAndExplicitWorkspaceConflict proves the default cwd and an
// explicit absolute spelling of the same directory resolve equal and conflict.
func TestDefaultAndExplicitWorkspaceConflict(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
cwd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
entered := make(chan struct{}, 2)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: cwd, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", first.ID)) {
t.Fatalf("default cwd and its explicit absolute path did not conflict: %v", err)
}
close(release)
r.wg.Wait()
}
// TestIdempotentRepeatWhileActive proves repeating one request key returns the
// same active run whether the run is in an explicit workspace or the default.
func TestIdempotentRepeatWhileActive(t *testing.T) {
_, r, pipelineID := concurrentStore(t)
entered := make(chan struct{}, 4)
release := make(chan struct{})
r.execute = blockingExecute(entered, release)
explicit := t.TempDir()
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
repeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "explicit", Workspace: explicit, Input: map[string]any{"prompt": "task"}})
if err != nil || repeat.ID != first.ID || !repeat.Executing {
t.Fatalf("explicit-workspace repeat was not idempotent: %+v %v", repeat, err)
}
// Default workspace: no Workspace field, resolved from the process cwd.
defaultFirst, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
defaultRepeat, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}})
if err != nil || defaultRepeat.ID != defaultFirst.ID || !defaultRepeat.Executing {
t.Fatalf("default-workspace repeat was not idempotent: %+v %v", defaultRepeat, err)
}
close(release)
r.wg.Wait()
}
// TestRestartRecoveryStillRefuses proves the existing guard is unchanged: an
// agent session left active with no in-memory active run blocks new work.
func TestRestartRecoveryStillRefuses(t *testing.T) {
ctx, r, pipelineID := concurrentStore(t)
d, err := r.store.Get(ctx, pipelineID, 1)
if err != nil {
t.Fatal(err)
}
workspace := t.TempDir()
id, _, err := r.store.CreateRun(ctx, d, "leftover", workspace, false, nil, map[string]any{"prompt": "task"})
if err != nil {
t.Fatal(err)
}
if err := r.store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "task"}); err != nil {
t.Fatal(err)
}
if err := r.store.RunSession(ctx, id, 1, "ses_leftover", true, "v2", ""); err != nil {
t.Fatal(err)
}
if len(r.active) != 0 {
t.Fatal("test expected no in-memory active runs")
}
_, err = r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "new", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err == nil || !strings.Contains(err.Error(), "restart recovery must interrupt it") {
t.Fatalf("restart-recovery refusal was not preserved: %v", err)
}
}
// TestRetryConflictsOnlyWithSameWorkspaceActiveRun proves retry forwards the
// original run's workspace and is refused only by an active run in that same
// workspace.
func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) {
ctx, r, pipelineID := concurrentStore(t)
a, b := t.TempDir(), t.TempDir()
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, fmt.Errorf("deliberate failure")
}
original, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "original", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
failed, err := r.store.GetRun(ctx, original.ID)
if err != nil || failed.Status != "failed" {
t.Fatalf("original run did not fail as intended: %+v %v", failed, err)
}
entered := make(chan struct{}, 4)
resolvedA, err := filepath.EvalSymlinks(a)
if err != nil {
t.Fatal(err)
}
releaseA := make(chan struct{})
releaseB := make(chan struct{})
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
entered <- struct{}{}
release := releaseB
if input["pwd"] == resolvedA {
release = releaseA
}
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
activeA, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-a", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
if _, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-a"}); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", activeA.ID)) || !strings.Contains(err.Error(), "different workspace") {
t.Fatalf("retry did not conflict with same-workspace active run: %v", err)
}
close(releaseA)
// Execution returning does not instantly remove the workspace reservation.
// Finish A's cleanup before asserting that only the unrelated B is active.
r.wg.Wait()
// A different active workspace must not block the retry of the original.
if _, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "active-b", Workspace: b, Input: map[string]any{"prompt": "task"}}); err != nil {
t.Fatal(err)
}
waitEntered(t, entered)
retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "retry-b"})
if err != nil {
t.Fatalf("retry was blocked by an unrelated workspace: %v", err)
}
if retried.Workspace != failed.Workspace {
t.Fatalf("retry did not reuse the original workspace: %q != %q", retried.Workspace, failed.Workspace)
}
close(releaseB)
r.wg.Wait()
}
// waitTerminal polls one run until it leaves queued/running, failing fast instead
// of hanging. It is scoped to the run so a blocked sibling does not stall it.
func waitTerminal(t *testing.T, r *runner, id int64) *protocol.RunDetail {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
detail, err := r.getRun(id)
if err != nil {
t.Fatal(err)
}
switch detail.Status {
case "succeeded", "failed", "canceled":
return detail
}
if time.Now().After(deadline) {
t.Fatalf("run %d did not reach a terminal state: %+v", id, detail)
}
time.Sleep(5 * time.Millisecond)
}
}
// TestOverlappingRunsDoNotFailQuiescence proves the step-completion check is
// scoped to the run: with two agent runs in different workspaces each recording
// its OWN session active, releasing run A lets A reach succeeded while run B's
// session is still active, and vice versa. A server-wide check would fail A
// deterministically against B's active session.
func TestOverlappingRunsDoNotFailQuiescence(t *testing.T) {
_, r, pipelineID := agentStore(t)
r.exportSession = validGateExport // Quiescence, not export transport, is under test.
a, b := t.TempDir(), t.TempDir()
type agentRun struct {
id int64
session string
}
var mu sync.Mutex
byWorkspace := map[string]agentRun{}
releases := map[string]chan struct{}{}
ready := make(chan struct{})
entered := make(chan string, 2)
resolve := func(dir string) string {
t.Helper()
resolved, err := filepath.EvalSymlinks(dir)
if err != nil {
t.Fatal(err)
}
return resolved
}
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
<-ready
workspace, _ := input["pwd"].(string)
mu.Lock()
agent, ok := byWorkspace[workspace]
release := releases[workspace]
mu.Unlock()
if !ok {
return stage.Result{}, fmt.Errorf("execute for unknown workspace %q", workspace)
}
if err := r.store.RunSession(context.Background(), agent.id, 1, agent.session, true, "v2", ""); err != nil {
return stage.Result{}, err
}
entered <- workspace
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
if err := r.store.RunSession(context.Background(), agent.id, 1, agent.session, false, "v2", ""); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatalf("second workspace was blocked by the first: %v", err)
}
if first.Workspace != resolve(a) || second.Workspace != resolve(b) {
t.Fatalf("unexpected resolved workspaces: %q, %q", first.Workspace, second.Workspace)
}
mu.Lock()
byWorkspace[first.Workspace] = agentRun{id: first.ID, session: "ses_a"}
byWorkspace[second.Workspace] = agentRun{id: second.ID, session: "ses_b"}
releases[first.Workspace] = make(chan struct{})
releases[second.Workspace] = make(chan struct{})
mu.Unlock()
close(ready)
waitEnteredString(t, entered)
waitEnteredString(t, entered)
for _, id := range []int64{first.ID, second.ID} {
detail, err := r.getRun(id)
if err != nil || !detail.Executing {
t.Fatalf("run %d was not reported executing: %v", id, err)
}
}
// Release A only; B's own session stays active while A finishes.
close(releases[first.Workspace])
done := waitTerminal(t, r, first.ID)
if done.Status != "succeeded" || strings.Contains(done.Diagnostic, "agent quiescence not confirmed") {
t.Fatalf("run A failed despite its own session being idle: %+v", done)
}
if detail, err := r.getRun(second.ID); err != nil || !detail.Executing {
t.Fatalf("run B stopped while A finished: %+v %v", detail, err)
}
close(releases[second.Workspace])
done = waitTerminal(t, r, second.ID)
if done.Status != "succeeded" || strings.Contains(done.Diagnostic, "agent quiescence not confirmed") {
t.Fatalf("run B failed despite its own session being idle: %+v", done)
}
r.wg.Wait()
}
// waitEnteredString fails fast instead of hanging when a stage never starts; it
// carries the entering workspace for tests that key on it.
func waitEnteredString(t *testing.T, entered <-chan string) {
t.Helper()
select {
case <-entered:
case <-time.After(10 * time.Second):
t.Fatal("stage execution did not start")
}
}
// Active ownership is not a terminal failure: the agent might still mutate files.
func TestOwnActiveSessionKeepsRunUnsettled(t *testing.T) {
_, r, pipelineID := agentStore(t)
workspace := t.TempDir()
ready := make(chan struct{})
var runID int64
r.execute = func(_ context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
<-ready
if err := r.store.RunSession(context.Background(), runID, 1, "ses_own", true, "v2", ""); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "own", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
runID = run.ID
close(ready)
r.wg.Wait()
detail, err := r.store.GetRun(context.Background(), run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "running" || !detail.CleanupUnconfirmed || detail.FinishedAt != "" {
t.Fatalf("own active session falsely settled the run: %+v", detail)
}
}