Files
pipelines/internal/server/review_contract_test.go
T
alan e70c94e8dd control: make advisory review findings non-blocking and bound retryable blocks
Cut the review loop's repeat cost and make a blocked stage actionable, at the
definition level only, with the runner's existing visit counting and caps.

The published maintenance, quick and focused pipelines gain a reviewer
`advisory -> next stage` edge for non-blocking notes, so an advisory-only review
continues the run instead of buying another implementer visit; `rejected ->
implementer` and its unchanged {"max_visits":3,"on":["rejected"]} limit keep a
must-fix finding looping and still terminate. The implementer gains
blocked_retryable -> implementer with {"max_visits":5,"on":["blocked_retryable"]}
for a transient environment/tooling/device problem a fresh attempt could
survive, while blocked -> #failure stays terminal. The cap is at least the
reviewer's rejection budget so the implementer's own pre-execution cap cannot
preempt the reviewer's rejected control.

No production Go code changed. New tests in internal/server/review_contract_test.go
cover advisory-only never revisiting the implementer, must-fix still looping to
the reviewer limit, the implementer cap not preempting the reviewer, retryable
blocks bounded then failing closed, a retry that recovers, and a terminal block
failing immediately; concurrency/control/human tests share the new submitRunner
and startRun seams.

Documentation checked: internal/author/skill/references/control.md and
docs/harnesses-and-control.md carry the new edges and the visit-cap rule;
no packaged template defines these pipelines (they are published immutable
versions), and docs/model-results.md needs no change because the {outcome,output}
result contract is unchanged. README and other shared docs deliberately untouched.

Verified: go build ./... and go test -count=1 ./... pass.

Residual limitation: the must-fix-vs-advisory split is carried by the prompt
contract, not a machine-readable marker, because the recorded findings have none;
a model that still returns rejected for advisory-only notes would still buy a
visit. All 18 recorded implementer blocked runs classify terminal, so the
blocked_retryable edge recovers 0 of them.
2026-10-17 17:06:51 +01:00

215 lines
11 KiB
Go

package server
import (
"context"
"strconv"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// reviewLoopRunner submits a definition through the parser and opens a runner
// over it, returning the runner, its store, the pipeline id and a per-stage call
// counter. It reuses the shared submitRunner seam so every assertion below runs
// the real runLoop, not a copy of its setup or routing.
func reviewLoopRunner(t *testing.T, ctx context.Context, source string) (*runner, *pipeline.Store, int64, map[string]int) {
t.Helper()
r, store, id := submitRunner(t, ctx, source)
t.Cleanup(func() { store.Close() })
return r, store, id, map[string]int{}
}
func getRun(t *testing.T, store *pipeline.Store, ctx context.Context, id int64) *protocol.RunDetail {
t.Helper()
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
return detail
}
// TestReviewerAdvisoryDoesNotRevisitImplementer proves an advisory-only review
// outcome moves the run forward and never buys another worker visit, while the
// rejected edge and its limits stay declared. The advisory route is the single
// definition-level mechanism that keeps a non-blocking rejection from looping.
func TestReviewerAdvisoryDoesNotRevisitImplementer(t *testing.T) {
ctx := context.Background()
r, store, id, calls := reviewLoopRunner(t, ctx, reviewerLoopSource())
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
calls[def.Key]++
if def.Key == "review" {
return stage.Result{Outcome: "advisory", Output: stage.Values{"findings": "non-blocking nits only"}}, nil
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil
}
run := startRun(t, r, id, "advisory")
detail := getRun(t, store, ctx, run.ID)
if calls["worker"] != 1 {
t.Fatalf("advisory review bought %d worker visits, want 1", calls["worker"])
}
if detail.Status != "succeeded" || len(detail.Steps) != 3 {
t.Fatalf("advisory route ended %s with %d steps, want succeeded with 3", detail.Status, len(detail.Steps))
}
if detail.Steps[1].Outcome != "advisory" || detail.Steps[1].Target != "commit" {
t.Fatalf("advisory step = %+v", detail.Steps[1])
}
}
// TestReviewerMustFixRejectionLoopsAndStopsAtLimit proves a rejection that names a
// real defect still routes back to the implementer and still terminates at the
// reviewer's limits cap with the retained diagnostic, so the rejection path is
// preserved. It runs the exact published combination - the implementer's own
// blocked_retryable limits block plus the reviewer's rejected -> implementer edge -
// rather than the reviewer limits alone.
func TestReviewerMustFixRejectionLoopsAndStopsAtLimit(t *testing.T) {
ctx := context.Background()
r, store, id, calls := reviewLoopRunner(t, ctx, publishedReviewLoopSource(5))
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
calls[def.Key]++
if def.Key == "reviewer" {
return stage.Result{Outcome: "rejected", Output: stage.Values{"findings": "unmet acceptance check at worker.go:12"}}, nil
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil
}
run := startRun(t, r, id, "mustfix")
detail := getRun(t, store, ctx, run.ID)
if calls["reviewer"] != 3 || calls["implementer"] != 3 || len(detail.Steps) != 6 {
t.Fatalf("must-fix loop = calls %v, %d steps", calls, len(detail.Steps))
}
last := detail.Steps[5]
if detail.Status != "failed" || last.Outcome != "rejected" || last.Control == nil || !last.Control.LimitReached || last.Target != protocol.FailureEnd {
t.Fatalf("limit did not terminate the must-fix loop: %+v", detail)
}
if last.Diagnostic != "Pipelines stopped reviewer after 3 visits on outcome rejected" {
t.Fatalf("limit diagnostic = %q", last.Diagnostic)
}
}
// TestImplementerVisitCapMustNotPreemptReviewerLimit pins the regression the
// published shape had: an implementer limits cap below the reviewer's rejection
// budget stops the loop on the implementer's pre-execution visit cap, so the
// reviewer's third visit and its "Pipelines stopped reviewer..." diagnostic are
// unreachable. The cap must be at least the reviewer's budget (the published
// pipelines use 5) or the reviewer's rejected control is dead.
func TestImplementerVisitCapMustNotPreemptReviewerLimit(t *testing.T) {
ctx := context.Background()
r, store, id, calls := reviewLoopRunner(t, ctx, publishedReviewLoopSource(2))
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
calls[def.Key]++
if def.Key == "reviewer" {
return stage.Result{Outcome: "rejected", Output: stage.Values{"findings": "unmet acceptance check"}}, nil
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil
}
run := startRun(t, r, id, "preempted")
detail := getRun(t, store, ctx, run.ID)
if calls["reviewer"] != 2 || calls["implementer"] != 2 || len(detail.Steps) != 4 {
t.Fatalf("implementer cap did not preempt the reviewer: calls %v, %d steps", calls, len(detail.Steps))
}
if detail.Status != "failed" || detail.Diagnostic != "stage implementer visit limit reached (2)" {
t.Fatalf("preempted limit diagnostic = %+v", detail)
}
}
// TestBlockedRetryableRetriesWithinBudgetThenStops proves a transient block is
// retried only inside its declared visit bound and then fails closed, so a
// retryable implementer block cannot become a retry storm.
func TestBlockedRetryableRetriesWithinBudgetThenStops(t *testing.T) {
ctx := context.Background()
r, store, id, calls := reviewLoopRunner(t, ctx, retryableBlockSource(2))
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
calls[def.Key]++
return stage.Result{Outcome: "blocked_retryable", Output: stage.Values{"blocked_reason": "device lock held"}}, nil
}
run := startRun(t, r, id, "retryable")
detail := getRun(t, store, ctx, run.ID)
if calls["worker"] != 2 || len(detail.Steps) != 2 {
t.Fatalf("retry bound = calls %v, %d steps, want 2", calls, len(detail.Steps))
}
last := detail.Steps[1]
if detail.Status != "failed" || last.Outcome != "blocked_retryable" || last.Control == nil || !last.Control.LimitReached || last.Target != protocol.FailureEnd {
t.Fatalf("retryable block not bounded: %+v", last)
}
if last.Diagnostic != "Pipelines stopped worker after 2 visits on outcome blocked_retryable" {
t.Fatalf("retry bound diagnostic = %q", last.Diagnostic)
}
}
// TestBlockedRetryableRetryCanRecover proves the bounded retry is useful: a stage
// that blocks once on a transient cause and succeeds on the retry continues to
// its normal next stage instead of failing the run.
func TestBlockedRetryableRetryCanRecover(t *testing.T) {
ctx := context.Background()
r, store, id, calls := reviewLoopRunner(t, ctx, retryableBlockSource(3))
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
calls[def.Key]++
if def.Key == "worker" && calls["worker"] == 1 {
return stage.Result{Outcome: "blocked_retryable", Output: stage.Values{"blocked_reason": "held device lock"}}, nil
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil
}
run := startRun(t, r, id, "recovered")
detail := getRun(t, store, ctx, run.ID)
if calls["worker"] != 2 || calls["review"] != 1 || detail.Status != "succeeded" {
t.Fatalf("retry did not recover: calls %v, status %s", calls, detail.Status)
}
}
// TestTerminalBlockedFailsImmediately proves a terminal block (an acceptance
// criterion another attempt cannot change) stops on the first visit with no
// retry and a meaningful run diagnostic.
func TestTerminalBlockedFailsImmediately(t *testing.T) {
ctx := context.Background()
source := strings.Replace(pipeline.Template([]string{"worker", "review"}, true), "success -> review", "success -> review\nblocked -> #failure", 1)
r, store, id, calls := reviewLoopRunner(t, ctx, source)
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
calls[def.Key]++
return stage.Result{Outcome: "blocked", Output: stage.Values{"blocked_reason": "absent capability"}}, nil
}
run := startRun(t, r, id, "terminal")
detail := getRun(t, store, ctx, run.ID)
if calls["worker"] != 1 || len(detail.Steps) != 1 {
t.Fatalf("terminal block was retried: calls %v, %d steps", calls, len(detail.Steps))
}
if detail.Status != "failed" || detail.Diagnostic != `stage worker returned outcome "blocked"` {
t.Fatalf("terminal block diagnostic = %+v", detail)
}
if detail.Steps[0].Target != protocol.FailureEnd {
t.Fatalf("terminal block target = %q", detail.Steps[0].Target)
}
}
// reviewerLoopSource is the worker/review/commit shape the maintenance pipelines
// use: the reviewer keeps its bounded rejection edge and gains an advisory edge
// that continues the run without another worker visit.
func reviewerLoopSource() string {
source := pipeline.Template([]string{"worker", "review", "commit"}, true)
source = strings.Replace(source, "# stage review", "# stage review\n\n```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```", 1)
return strings.Replace(source, "success -> commit", "success -> commit\nadvisory -> commit\nrejected -> worker", 1)
}
// publishedReviewLoopSource reconstructs the published maintenance/quick/focused
// control shape with the real stage keys: the implementer carries its own
// blocked_retryable limits block and self-edge, and the reviewer's rejected edge
// returns to that same implementer. implementerVisits is the implementer's
// max_visits, which also caps its re-entries from the reviewer, so it must be at
// least the reviewer's rejection budget.
func publishedReviewLoopSource(implementerVisits int) string {
source := pipeline.Template([]string{"implementer", "reviewer", "commit"}, true)
source = strings.Replace(source, "# stage implementer", "# stage implementer\n\n```limits\n{\"max_visits\":"+strconv.Itoa(implementerVisits)+",\"on\":[\"blocked_retryable\"]}\n```", 1)
source = strings.Replace(source, "success -> reviewer", "success -> reviewer\nblocked_retryable -> implementer\nblocked -> #failure", 1)
source = strings.Replace(source, "# stage reviewer", "# stage reviewer\n\n```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```", 1)
return strings.Replace(source, "success -> commit", "success -> commit\nadvisory -> commit\nrejected -> implementer", 1)
}
// retryableBlockSource is a worker/review shape whose worker declares a bounded
// blocked_retryable self-edge, mirroring the published definitions.
func retryableBlockSource(maxVisits int) string {
source := pipeline.Template([]string{"worker", "review"}, true)
source = strings.Replace(source, "# stage worker", "# stage worker\n\n```limits\n{\"max_visits\":"+strconv.Itoa(maxVisits)+",\"on\":[\"blocked_retryable\"]}\n```", 1)
return strings.Replace(source, "success -> review", "success -> review\nblocked_retryable -> worker", 1)
}