server: no invented verdict reason for a turn body
The one-line verdict summariser expects the old field map; given a turn's single
markdown body it treated the result as untrusted reference material and produced
a rationale unrelated to the work ("The text was treated as untrusted..."). Turn
outputs now keep the honest absent-reason state: the card still shows the option,
and no model call is spent inventing a reason.
This commit is contained in:
@@ -1314,6 +1314,21 @@ func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage {
|
||||
// cannot drift between them. A persist or memory-copy failure is returned for
|
||||
// the caller to fail the run; the caller still owns the model-specific
|
||||
// verdict, quiescence and cancellation handling around it.
|
||||
// turnOutputOnly reports the turn protocol's single markdown body. The verdict
|
||||
// summariser is written for the old field map (summary/option/reason); given a
|
||||
// document it treats it as untrusted reference material and invents a reason.
|
||||
func turnOutputOnly(output stage.Values) bool {
|
||||
if len(output) != 1 {
|
||||
return false
|
||||
}
|
||||
body, ok := output["markdown"]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, isText := body.(string)
|
||||
return isText
|
||||
}
|
||||
|
||||
func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) {
|
||||
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
persistErr := r.store.FinishStep(save, id, step)
|
||||
@@ -1769,8 +1784,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
// Generate the one-line verdict reason once, at completion. It is
|
||||
// best-effort: a summariser failure or timeout returns NoSummary and must
|
||||
// never fail the stage or the run, so every error is swallowed. A failed
|
||||
// step still gets a verdict from its error envelope.
|
||||
if r.verdict != nil {
|
||||
// step still gets a verdict from its error envelope. A turn output is one
|
||||
// markdown body, which the field-map summariser reads as untrusted data and
|
||||
// answers with an invented rationale; those steps get the honest read-time
|
||||
// fallback instead.
|
||||
if r.verdict != nil && !turnOutputOnly(step.Output) {
|
||||
reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout)
|
||||
if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary {
|
||||
step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel}
|
||||
|
||||
@@ -364,6 +364,54 @@ func TestStepVerdictGeneratedOnceAtCompletion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestTurnOutputSkipsTheVerdictSummariser proves a turn's markdown body is not
|
||||
// handed to the field-map summariser, which would hallucinate a rationale; the
|
||||
// step keeps the honest read-time fallback instead.
|
||||
func TestTurnOutputSkipsTheVerdictSummariser(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "turn-verdict.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if _, err := store.Submit(ctx, "turn-verdict", pipeline.Template([]string{"one"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(ctx)
|
||||
r, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "## Result\n\nthe work is done"}}, nil
|
||||
}
|
||||
calls := 0
|
||||
r.verdict = func(context.Context, map[string]any) string {
|
||||
calls++
|
||||
return "invented rationale"
|
||||
}
|
||||
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "turn-verdict", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
if calls != 0 {
|
||||
t.Fatalf("the verdict summariser ran %d times for a turn output", calls)
|
||||
}
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verdict := detail.Steps[0].Verdict
|
||||
if verdict == nil || verdict.ReasonSource != protocol.VerdictReasonAbsent || verdict.Reason != "" {
|
||||
t.Fatalf("stored verdict = %+v, want the honest absent-reason state", verdict)
|
||||
}
|
||||
if verdict.Option != "success → success" {
|
||||
t.Fatalf("turn verdict option = %q", verdict.Option)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStepVerdictFailureNeverFailsStage proves a best-effort failure (the
|
||||
// NoSummary sentinel) still lets the run reach its normal terminal status, with
|
||||
// the verdict derived at read time from the step's output.
|
||||
|
||||
Reference in New Issue
Block a user