diff --git a/internal/server/runner.go b/internal/server/runner.go index 88668d21..22bddf34 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -1314,6 +1314,21 @@ func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage { // cannot drift between them. A persist or memory-copy failure is returned for // the caller to fail the run; the caller still owns the model-specific // verdict, quiescence and cancellation handling around it. +// turnOutputOnly reports the turn protocol's single markdown body. The verdict +// summariser is written for the old field map (summary/option/reason); given a +// document it treats it as untrusted reference material and invents a reason. +func turnOutputOnly(output stage.Values) bool { + if len(output) != 1 { + return false + } + body, ok := output["markdown"] + if !ok { + return false + } + _, isText := body.(string) + return isText +} + func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) { save, cancel := context.WithTimeout(context.Background(), 10*time.Second) persistErr := r.store.FinishStep(save, id, step) @@ -1769,8 +1784,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na // Generate the one-line verdict reason once, at completion. It is // best-effort: a summariser failure or timeout returns NoSummary and must // never fail the stage or the run, so every error is swallowed. A failed - // step still gets a verdict from its error envelope. - if r.verdict != nil { + // step still gets a verdict from its error envelope. A turn output is one + // markdown body, which the field-map summariser reads as untrusted data and + // answers with an invented rationale; those steps get the honest read-time + // fallback instead. + if r.verdict != nil && !turnOutputOnly(step.Output) { reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout) if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary { step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel} diff --git a/internal/server/runner_test.go b/internal/server/runner_test.go index a5d924e1..31b8921a 100644 --- a/internal/server/runner_test.go +++ b/internal/server/runner_test.go @@ -364,6 +364,54 @@ func TestStepVerdictGeneratedOnceAtCompletion(t *testing.T) { } } +// TestTurnOutputSkipsTheVerdictSummariser proves a turn's markdown body is not +// handed to the field-map summariser, which would hallucinate a rationale; the +// step keeps the honest read-time fallback instead. +func TestTurnOutputSkipsTheVerdictSummariser(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "turn-verdict.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "turn-verdict", pipeline.Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, _ := store.List(ctx) + r, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "## Result\n\nthe work is done"}}, nil + } + calls := 0 + r.verdict = func(context.Context, map[string]any) string { + calls++ + return "invented rationale" + } + run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "turn-verdict", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + if calls != 0 { + t.Fatalf("the verdict summariser ran %d times for a turn output", calls) + } + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + verdict := detail.Steps[0].Verdict + if verdict == nil || verdict.ReasonSource != protocol.VerdictReasonAbsent || verdict.Reason != "" { + t.Fatalf("stored verdict = %+v, want the honest absent-reason state", verdict) + } + if verdict.Option != "success → success" { + t.Fatalf("turn verdict option = %q", verdict.Option) + } +} + // TestStepVerdictFailureNeverFailsStage proves a best-effort failure (the // NoSummary sentinel) still lets the run reach its normal terminal status, with // the verdict derived at read time from the step's output.