package server import ( "context" "errors" "fmt" "path/filepath" "strings" "testing" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" "git.a-wan.net/alan/pipelines/internal/summarise" ) // TestRunTitleGeneratedPersistedAndFallback proves the insert path stores a // generated title with its provenance in list and detail responses, and falls // back to a derived bounded title when no generator is wired, without failing // run creation. func TestRunTitleGeneratedPersistedAndFallback(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "titles.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "titled", pipeline.Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } r.title = func(context.Context, string) summarise.Title { return summarise.Title{Text: "Fix the flaky login test", Source: summarise.SourceModel, Model: "openai/gpt"} } run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "model-title", Workspace: t.TempDir(), Input: map[string]any{"prompt": "prompt body"}}) if err != nil { t.Fatal(err) } r.wg.Wait() detail, err := store.GetRun(ctx, run.ID) if err != nil { t.Fatal(err) } if detail.Title != "Fix the flaky login test" || detail.TitleSource != summarise.SourceModel || detail.TitleModel != "openai/gpt" { t.Fatalf("model title not stored with provenance: %+v", detail) } list, err := store.ListRuns(ctx) if err != nil || len(list) != 1 || list[0].Title != "Fix the flaky login test" { t.Fatalf("run list omitted the title: %v %v", list, err) } // No generator: the fallback is derived from the prompt's first line and run // creation still succeeds. r.title = nil fallback, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "fallback-title", Workspace: t.TempDir(), Input: map[string]any{"prompt": "Fix the widget\nand more detail"}}) if err != nil { t.Fatal(err) } r.wg.Wait() detail, err = store.GetRun(ctx, fallback.ID) if err != nil { t.Fatal(err) } if detail.Title != "Fix the widget" || detail.TitleSource != summarise.SourceFallback || detail.TitleModel != "" { t.Fatalf("fallback title = %+v, want the prompt's first line", detail) } } // TestRunTitleGeneratorSkippedForReplayAndRejection proves the bounded // generation runs only for a request that starts a run: an idempotent replay and // a request refused for an already-reserved workspace must not pay for it. func TestRunTitleGeneratorSkippedForReplayAndRejection(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "skip.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "skip", pipeline.Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } release := make(chan struct{}) r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { <-release return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } calls := 0 r.title = func(context.Context, string) summarise.Title { calls++ return summarise.Title{Text: "Generated", Source: summarise.SourceModel} } workspace := t.TempDir() first, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "skip-key", Workspace: workspace, Input: map[string]any{"prompt": "first"}}) if err != nil { t.Fatal(err) } if calls != 1 { t.Fatalf("first start generated %d titles, want 1", calls) } // A replay while the first run is active reuses the stored title. replay, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "skip-key", Workspace: workspace, Input: map[string]any{"prompt": "first"}}) if err != nil || replay.ID != first.ID { t.Fatalf("replay = %+v, %v; want run %d", replay, err, first.ID) } // A new key in the already-reserved workspace is refused. if _, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "skip-other", Workspace: workspace, Input: map[string]any{"prompt": "second"}}); err == nil { t.Fatal("workspace conflict was not refused") } if calls != 1 { t.Fatalf("replay/rejection generated %d titles, want 1", calls) } close(release) r.wg.Wait() } func TestHandlerWiresRunTitleGenerator(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) store, err := pipeline.Open(filepath.Join(t.TempDir(), "wire.db")) if err != nil { t.Fatal(err) } defer store.Close() configured := HandlerWithOptions(ctx, Options{RunTitles: summarise.GenerateTitle}, store).(*runHandler) bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler) defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }() if configured.runner.title == nil { t.Fatal("configured RunTitles was not applied to the runner") } if bare.runner.title != nil { t.Fatal("unconfigured RunTitles must keep deterministic fallback title generation") } } // TestRunnerExplicitTitleSkipsGeneratorAndRenames proves an explicit --title is // stored verbatim and skips the model generator, and that setTitle later // replaces it with an explicit provenance without touching the run's input. func TestRunnerExplicitTitleSkipsGeneratorAndRenames(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "explicit.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "explicit", pipeline.Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } generated := 0 r.title = func(context.Context, string) summarise.Title { generated++ return summarise.Title{Text: "Generated", Source: summarise.SourceModel} } run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "explicit-key", Workspace: t.TempDir(), Title: "Operator chosen title", Input: map[string]any{"prompt": "prompt body"}}) if err != nil { t.Fatal(err) } r.wg.Wait() if generated != 0 { t.Fatalf("explicit title still called the generator %d times", generated) } if run.Title != "Operator chosen title" || run.TitleSource != summarise.SourceExplicit || run.TitleModel != "" { t.Fatalf("explicit title = %+v", run) } renamed, err := r.setTitle(run.ID, " Renamed after the fact ") if err != nil { t.Fatal(err) } if renamed.Title != "Renamed after the fact" || renamed.TitleSource != summarise.SourceExplicit || renamed.Input["prompt"] != "prompt body" { t.Fatalf("rename = %+v", renamed) } if _, err := r.setTitle(run.ID, " "); err == nil { t.Fatal("blank rename was accepted") } if _, err := r.setTitle(run.ID+999, "ghost"); err == nil { t.Fatal("renaming an unknown run was accepted") } } func TestRunnerRoutingPersistenceAndIdempotency(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "runs.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "test", pipeline.Template([]string{"one", "two"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } entered := make(chan struct{}) release := make(chan struct{}) r.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) { if def.Key == "one" { if len(input) != 2 || input["prompt"] != "task" || input["pwd"] == "" { t.Error("entry input must be the user prompt unchanged") } close(entered) <-release return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "evidence"}}, nil } if len(input) != 1 || input["handoff"] != "evidence" { t.Error("handoff not propagated") } return stage.Result{Outcome: "success", Output: stage.Values{"verified": true}}, nil } request := protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "same", Input: map[string]any{"prompt": "task"}} run, err := r.start(request) if err != nil { t.Fatal(err) } <-entered duplicate, err := r.start(request) if err != nil || duplicate.ID != run.ID { t.Fatal("active repeat was not idempotent", err) } other := request other.RequestKey = "other" if _, err := r.start(other); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", run.ID)) || !strings.Contains(err.Error(), "different workspace") { t.Fatal("overlapping execution was not refused with the conflicting run", err) } close(release) r.wg.Wait() detail, err := store.GetRun(ctx, run.ID) if err != nil { t.Fatal(err) } if detail.Status != "succeeded" || len(detail.Steps) != 2 || detail.Steps[1].Input["handoff"] != "evidence" || detail.Output["verified"] != true { t.Fatalf("invalid persisted run: %+v", detail) } duplicate, err = r.start(request) if err != nil || duplicate.ID != run.ID { t.Fatal("finished repeat not idempotent") } request.RequestKey = "failure" r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{}, errors.New("provider failed") } failed, err := r.start(request) if err != nil { t.Fatal(err) } r.wg.Wait() detail, _ = store.GetRun(ctx, failed.ID) if detail.Status != "failed" || detail.Steps[0].Outcome != "error" || detail.Steps[0].Target != protocol.FailureEnd { t.Fatal("execution error did not follow mandatory error edge") } request.RequestKey = "cancel" r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { <-ctx.Done() return stage.Result{}, ctx.Err() } canceled, err := r.start(request) if err != nil { t.Fatal(err) } if err := r.cancel(canceled.ID); err != nil { t.Fatal(err) } r.wg.Wait() detail, _ = store.GetRun(ctx, canceled.ID) if detail.Status != "canceled" { t.Fatal("cancel not persisted") } request.RequestKey = "budget" r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } // A new version with a self-loop remains reachable to a terminal via error. source := pipeline.Template([]string{"loop"}, true) source = strings.Replace(source, "success -> #success", "success -> loop", 1) if _, err := store.Submit(ctx, "loop", source); err != nil { t.Fatal(err) } items, _ = store.List(ctx) for _, p := range items { if p.Name == "loop" { request.PipelineID = p.ID } } bounded, err := r.start(request) if err != nil { t.Fatal(err) } r.wg.Wait() detail, _ = store.GetRun(ctx, bounded.ID) if detail.Status != "failed" || len(detail.Steps) != 5 { t.Fatalf("cycle not bounded: %+v", detail) } } // TestStepVerdictGeneratedOnceAtCompletion proves the wired seam runs exactly // once per completed stage visit, from the step's own output, and the generated // line is stored with model provenance. func TestStepVerdictGeneratedOnceAtCompletion(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "verdict.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "verdict", pipeline.Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{"summary": "ran the checks"}}, nil } calls := 0 r.verdict = func(_ context.Context, output map[string]any) string { calls++ if output["summary"] != "ran the checks" { t.Errorf("verdict seam got output %v", output) } return "chosen because every check passed" } run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "verdict-once", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } r.wg.Wait() if calls != 1 { t.Fatalf("verdict seam ran %d times, want exactly once for one step", calls) } detail, err := store.GetRun(ctx, run.ID) if err != nil { t.Fatal(err) } verdict := detail.Steps[0].Verdict if verdict == nil || verdict.Reason != "chosen because every check passed" || verdict.ReasonSource != protocol.VerdictReasonModel { t.Fatalf("stored verdict = %+v", verdict) } } // TestStepVerdictFailureNeverFailsStage proves a best-effort failure (the // NoSummary sentinel) still lets the run reach its normal terminal status, with // the verdict derived at read time from the step's output. func TestStepVerdictFailureNeverFailsStage(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "verdict-fail.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "verdict-fail", pipeline.Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{"summary": "ran the checks"}}, nil } r.verdict = func(context.Context, map[string]any) string { return summarise.NoSummary } run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "verdict-fail", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } r.wg.Wait() detail, err := store.GetRun(ctx, run.ID) if err != nil { t.Fatal(err) } if detail.Status != "succeeded" { t.Fatalf("summariser failure changed the run status: %+v", detail) } verdict := detail.Steps[0].Verdict if verdict == nil || verdict.Reason != "ran the checks" || verdict.ReasonSource != protocol.VerdictReasonFallback { t.Fatalf("read-time verdict after a summariser failure = %+v", verdict) } } func TestHandlerWiresStepVerdictGenerator(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) store, err := pipeline.Open(filepath.Join(t.TempDir(), "verdict-wire.db")) if err != nil { t.Fatal(err) } defer store.Close() configured := HandlerWithOptions(ctx, Options{StepVerdicts: summarise.GenerateVerdictReason}, store).(*runHandler) bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler) defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }() if configured.runner.verdict == nil { t.Fatal("configured StepVerdicts was not applied to the runner") } if bare.runner.verdict != nil { t.Fatal("unconfigured StepVerdicts must keep deterministic read-time derivation") } }