5bb96cd4dc
The one-line verdict summariser expects the old field map; given a turn's single
markdown body it treated the result as untrusted reference material and produced
a rationale unrelated to the work ("The text was treated as untrusted..."). Turn
outputs now keep the honest absent-reason state: the card still shows the option,
and no model call is spent inventing a reason.
473 lines
17 KiB
Go
473 lines
17 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
)
|
|
|
|
// TestRunTitleGeneratedPersistedAndFallback proves the insert path stores a
|
|
// generated title with its provenance in list and detail responses, and falls
|
|
// back to a derived bounded title when no generator is wired, without failing
|
|
// run creation.
|
|
func TestRunTitleGeneratedPersistedAndFallback(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "titles.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "titled", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
r.title = func(context.Context, string) summarise.Title {
|
|
return summarise.Title{Text: "Fix the flaky login test", Source: summarise.SourceModel, Model: "openai/gpt"}
|
|
}
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "model-title", Workspace: t.TempDir(), Input: map[string]any{"prompt": "prompt body"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Title != "Fix the flaky login test" || detail.TitleSource != summarise.SourceModel || detail.TitleModel != "openai/gpt" {
|
|
t.Fatalf("model title not stored with provenance: %+v", detail)
|
|
}
|
|
list, err := store.ListRuns(ctx)
|
|
if err != nil || len(list) != 1 || list[0].Title != "Fix the flaky login test" {
|
|
t.Fatalf("run list omitted the title: %v %v", list, err)
|
|
}
|
|
|
|
// No generator: the fallback is derived from the prompt's first line and run
|
|
// creation still succeeds.
|
|
r.title = nil
|
|
fallback, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "fallback-title", Workspace: t.TempDir(), Input: map[string]any{"prompt": "Fix the widget\nand more detail"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err = store.GetRun(ctx, fallback.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Title != "Fix the widget" || detail.TitleSource != summarise.SourceFallback || detail.TitleModel != "" {
|
|
t.Fatalf("fallback title = %+v, want the prompt's first line", detail)
|
|
}
|
|
}
|
|
|
|
// TestRunTitleGeneratorSkippedForReplayAndRejection proves the bounded
|
|
// generation runs only for a request that starts a run: an idempotent replay and
|
|
// a request refused for an already-reserved workspace must not pay for it.
|
|
func TestRunTitleGeneratorSkippedForReplayAndRejection(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "skip.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "skip", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
release := make(chan struct{})
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
<-release
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
calls := 0
|
|
r.title = func(context.Context, string) summarise.Title {
|
|
calls++
|
|
return summarise.Title{Text: "Generated", Source: summarise.SourceModel}
|
|
}
|
|
workspace := t.TempDir()
|
|
first, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "skip-key", Workspace: workspace, Input: map[string]any{"prompt": "first"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if calls != 1 {
|
|
t.Fatalf("first start generated %d titles, want 1", calls)
|
|
}
|
|
// A replay while the first run is active reuses the stored title.
|
|
replay, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "skip-key", Workspace: workspace, Input: map[string]any{"prompt": "first"}})
|
|
if err != nil || replay.ID != first.ID {
|
|
t.Fatalf("replay = %+v, %v; want run %d", replay, err, first.ID)
|
|
}
|
|
// A new key in the already-reserved workspace is refused.
|
|
if _, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "skip-other", Workspace: workspace, Input: map[string]any{"prompt": "second"}}); err == nil {
|
|
t.Fatal("workspace conflict was not refused")
|
|
}
|
|
if calls != 1 {
|
|
t.Fatalf("replay/rejection generated %d titles, want 1", calls)
|
|
}
|
|
close(release)
|
|
r.wg.Wait()
|
|
}
|
|
|
|
func TestHandlerWiresRunTitleGenerator(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "wire.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
configured := HandlerWithOptions(ctx, Options{RunTitles: summarise.GenerateTitle}, store).(*runHandler)
|
|
bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler)
|
|
defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }()
|
|
if configured.runner.title == nil {
|
|
t.Fatal("configured RunTitles was not applied to the runner")
|
|
}
|
|
if bare.runner.title != nil {
|
|
t.Fatal("unconfigured RunTitles must keep deterministic fallback title generation")
|
|
}
|
|
}
|
|
|
|
// TestRunnerExplicitTitleSkipsGeneratorAndRenames proves an explicit --title is
|
|
// stored verbatim and skips the model generator, and that setTitle later
|
|
// replaces it with an explicit provenance without touching the run's input.
|
|
func TestRunnerExplicitTitleSkipsGeneratorAndRenames(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "explicit.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "explicit", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
generated := 0
|
|
r.title = func(context.Context, string) summarise.Title {
|
|
generated++
|
|
return summarise.Title{Text: "Generated", Source: summarise.SourceModel}
|
|
}
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "explicit-key", Workspace: t.TempDir(), Title: "Operator chosen title", Input: map[string]any{"prompt": "prompt body"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if generated != 0 {
|
|
t.Fatalf("explicit title still called the generator %d times", generated)
|
|
}
|
|
if run.Title != "Operator chosen title" || run.TitleSource != summarise.SourceExplicit || run.TitleModel != "" {
|
|
t.Fatalf("explicit title = %+v", run)
|
|
}
|
|
|
|
renamed, err := r.setTitle(run.ID, " Renamed after the fact ")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if renamed.Title != "Renamed after the fact" || renamed.TitleSource != summarise.SourceExplicit || renamed.Input["prompt"] != "prompt body" {
|
|
t.Fatalf("rename = %+v", renamed)
|
|
}
|
|
if _, err := r.setTitle(run.ID, " "); err == nil {
|
|
t.Fatal("blank rename was accepted")
|
|
}
|
|
if _, err := r.setTitle(run.ID+999, "ghost"); err == nil {
|
|
t.Fatal("renaming an unknown run was accepted")
|
|
}
|
|
}
|
|
|
|
func TestRunnerRoutingPersistenceAndIdempotency(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "runs.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "test", pipeline.Template([]string{"one", "two"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entered := make(chan struct{})
|
|
release := make(chan struct{})
|
|
r.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
|
if def.Key == "one" {
|
|
if len(input) != 2 || input["prompt"] != "task" || input["pwd"] == "" {
|
|
t.Error("entry input must be the user prompt unchanged")
|
|
}
|
|
close(entered)
|
|
<-release
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "evidence"}}, nil
|
|
}
|
|
if len(input) != 1 || input["handoff"] != "evidence" {
|
|
t.Error("handoff not propagated")
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"verified": true}}, nil
|
|
}
|
|
request := protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "same", Input: map[string]any{"prompt": "task"}}
|
|
run, err := r.start(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
<-entered
|
|
duplicate, err := r.start(request)
|
|
if err != nil || duplicate.ID != run.ID {
|
|
t.Fatal("active repeat was not idempotent", err)
|
|
}
|
|
other := request
|
|
other.RequestKey = "other"
|
|
if _, err := r.start(other); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", run.ID)) || !strings.Contains(err.Error(), "different workspace") {
|
|
t.Fatal("overlapping execution was not refused with the conflicting run", err)
|
|
}
|
|
close(release)
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 2 || detail.Steps[1].Input["handoff"] != "evidence" || detail.Output["verified"] != true {
|
|
t.Fatalf("invalid persisted run: %+v", detail)
|
|
}
|
|
duplicate, err = r.start(request)
|
|
if err != nil || duplicate.ID != run.ID {
|
|
t.Fatal("finished repeat not idempotent")
|
|
}
|
|
request.RequestKey = "failure"
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{}, errors.New("provider failed")
|
|
}
|
|
failed, err := r.start(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, _ = store.GetRun(ctx, failed.ID)
|
|
if detail.Status != "failed" || detail.Steps[0].Outcome != "error" || detail.Steps[0].Target != protocol.FailureEnd {
|
|
t.Fatal("execution error did not follow mandatory error edge")
|
|
}
|
|
request.RequestKey = "cancel"
|
|
r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
<-ctx.Done()
|
|
return stage.Result{}, ctx.Err()
|
|
}
|
|
canceled, err := r.start(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.cancel(canceled.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, _ = store.GetRun(ctx, canceled.ID)
|
|
if detail.Status != "canceled" {
|
|
t.Fatal("cancel not persisted")
|
|
}
|
|
request.RequestKey = "budget"
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
// A new version with a self-loop remains reachable to a terminal via error.
|
|
source := pipeline.Template([]string{"loop"}, true)
|
|
source = strings.Replace(source, "success -> #success", "success -> loop", 1)
|
|
if _, err := store.Submit(ctx, "loop", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ = store.List(ctx)
|
|
for _, p := range items {
|
|
if p.Name == "loop" {
|
|
request.PipelineID = p.ID
|
|
}
|
|
}
|
|
bounded, err := r.start(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, _ = store.GetRun(ctx, bounded.ID)
|
|
if detail.Status != "failed" || len(detail.Steps) != 5 {
|
|
t.Fatalf("cycle not bounded: %+v", detail)
|
|
}
|
|
}
|
|
|
|
// TestStepVerdictGeneratedOnceAtCompletion proves the wired seam runs exactly
|
|
// once per completed stage visit, from the step's own output, and the generated
|
|
// line is stored with model provenance.
|
|
func TestStepVerdictGeneratedOnceAtCompletion(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "verdict.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "verdict", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "ran the checks"}}, nil
|
|
}
|
|
calls := 0
|
|
r.verdict = func(_ context.Context, output map[string]any) string {
|
|
calls++
|
|
if output["summary"] != "ran the checks" {
|
|
t.Errorf("verdict seam got output %v", output)
|
|
}
|
|
return "chosen because every check passed"
|
|
}
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "verdict-once", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if calls != 1 {
|
|
t.Fatalf("verdict seam ran %d times, want exactly once for one step", calls)
|
|
}
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
verdict := detail.Steps[0].Verdict
|
|
if verdict == nil || verdict.Reason != "chosen because every check passed" || verdict.ReasonSource != protocol.VerdictReasonModel {
|
|
t.Fatalf("stored verdict = %+v", verdict)
|
|
}
|
|
}
|
|
|
|
// TestTurnOutputSkipsTheVerdictSummariser proves a turn's markdown body is not
|
|
// handed to the field-map summariser, which would hallucinate a rationale; the
|
|
// step keeps the honest read-time fallback instead.
|
|
func TestTurnOutputSkipsTheVerdictSummariser(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "turn-verdict.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "turn-verdict", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "## Result\n\nthe work is done"}}, nil
|
|
}
|
|
calls := 0
|
|
r.verdict = func(context.Context, map[string]any) string {
|
|
calls++
|
|
return "invented rationale"
|
|
}
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "turn-verdict", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if calls != 0 {
|
|
t.Fatalf("the verdict summariser ran %d times for a turn output", calls)
|
|
}
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
verdict := detail.Steps[0].Verdict
|
|
if verdict == nil || verdict.ReasonSource != protocol.VerdictReasonAbsent || verdict.Reason != "" {
|
|
t.Fatalf("stored verdict = %+v, want the honest absent-reason state", verdict)
|
|
}
|
|
if verdict.Option != "success → success" {
|
|
t.Fatalf("turn verdict option = %q", verdict.Option)
|
|
}
|
|
}
|
|
|
|
// TestStepVerdictFailureNeverFailsStage proves a best-effort failure (the
|
|
// NoSummary sentinel) still lets the run reach its normal terminal status, with
|
|
// the verdict derived at read time from the step's output.
|
|
func TestStepVerdictFailureNeverFailsStage(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "verdict-fail.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "verdict-fail", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "ran the checks"}}, nil
|
|
}
|
|
r.verdict = func(context.Context, map[string]any) string { return summarise.NoSummary }
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "verdict-fail", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "succeeded" {
|
|
t.Fatalf("summariser failure changed the run status: %+v", detail)
|
|
}
|
|
verdict := detail.Steps[0].Verdict
|
|
if verdict == nil || verdict.Reason != "ran the checks" || verdict.ReasonSource != protocol.VerdictReasonFallback {
|
|
t.Fatalf("read-time verdict after a summariser failure = %+v", verdict)
|
|
}
|
|
}
|
|
|
|
func TestHandlerWiresStepVerdictGenerator(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "verdict-wire.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
configured := HandlerWithOptions(ctx, Options{StepVerdicts: summarise.GenerateVerdictReason}, store).(*runHandler)
|
|
bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler)
|
|
defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }()
|
|
if configured.runner.verdict == nil {
|
|
t.Fatal("configured StepVerdicts was not applied to the runner")
|
|
}
|
|
if bare.runner.verdict != nil {
|
|
t.Fatal("unconfigured StepVerdicts must keep deterministic read-time derivation")
|
|
}
|
|
}
|