server: no invented verdict reason for a turn body

The one-line verdict summariser expects the old field map; given a turn's single
markdown body it treated the result as untrusted reference material and produced
a rationale unrelated to the work ("The text was treated as untrusted..."). Turn
outputs now keep the honest absent-reason state: the card still shows the option,
and no model call is spent inventing a reason.
This commit is contained in:
2026-10-19 23:41:34 +01:00
parent 00252da8f8
commit 5bb96cd4dc
2 changed files with 68 additions and 2 deletions
+20 -2
View File
@@ -1314,6 +1314,21 @@ func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage {
// cannot drift between them. A persist or memory-copy failure is returned for
// the caller to fail the run; the caller still owns the model-specific
// verdict, quiescence and cancellation handling around it.
// turnOutputOnly reports the turn protocol's single markdown body. The verdict
// summariser is written for the old field map (summary/option/reason); given a
// document it treats it as untrusted reference material and invents a reason.
func turnOutputOnly(output stage.Values) bool {
if len(output) != 1 {
return false
}
body, ok := output["markdown"]
if !ok {
return false
}
_, isText := body.(string)
return isText
}
func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) {
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
persistErr := r.store.FinishStep(save, id, step)
@@ -1769,8 +1784,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
// Generate the one-line verdict reason once, at completion. It is
// best-effort: a summariser failure or timeout returns NoSummary and must
// never fail the stage or the run, so every error is swallowed. A failed
// step still gets a verdict from its error envelope.
if r.verdict != nil {
// step still gets a verdict from its error envelope. A turn output is one
// markdown body, which the field-map summariser reads as untrusted data and
// answers with an invented rationale; those steps get the honest read-time
// fallback instead.
if r.verdict != nil && !turnOutputOnly(step.Output) {
reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout)
if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary {
step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel}
+48
View File
@@ -364,6 +364,54 @@ func TestStepVerdictGeneratedOnceAtCompletion(t *testing.T) {
}
}
// TestTurnOutputSkipsTheVerdictSummariser proves a turn's markdown body is not
// handed to the field-map summariser, which would hallucinate a rationale; the
// step keeps the honest read-time fallback instead.
func TestTurnOutputSkipsTheVerdictSummariser(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "turn-verdict.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "turn-verdict", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "## Result\n\nthe work is done"}}, nil
}
calls := 0
r.verdict = func(context.Context, map[string]any) string {
calls++
return "invented rationale"
}
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "turn-verdict", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if calls != 0 {
t.Fatalf("the verdict summariser ran %d times for a turn output", calls)
}
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
verdict := detail.Steps[0].Verdict
if verdict == nil || verdict.ReasonSource != protocol.VerdictReasonAbsent || verdict.Reason != "" {
t.Fatalf("stored verdict = %+v, want the honest absent-reason state", verdict)
}
if verdict.Option != "success → success" {
t.Fatalf("turn verdict option = %q", verdict.Option)
}
}
// TestStepVerdictFailureNeverFailsStage proves a best-effort failure (the
// NoSummary sentinel) still lets the run reach its normal terminal status, with
// the verdict derived at read time from the step's output.