e70c94e8dd
Cut the review loop's repeat cost and make a blocked stage actionable, at the
definition level only, with the runner's existing visit counting and caps.
The published maintenance, quick and focused pipelines gain a reviewer
`advisory -> next stage` edge for non-blocking notes, so an advisory-only review
continues the run instead of buying another implementer visit; `rejected ->
implementer` and its unchanged {"max_visits":3,"on":["rejected"]} limit keep a
must-fix finding looping and still terminate. The implementer gains
blocked_retryable -> implementer with {"max_visits":5,"on":["blocked_retryable"]}
for a transient environment/tooling/device problem a fresh attempt could
survive, while blocked -> #failure stays terminal. The cap is at least the
reviewer's rejection budget so the implementer's own pre-execution cap cannot
preempt the reviewer's rejected control.
No production Go code changed. New tests in internal/server/review_contract_test.go
cover advisory-only never revisiting the implementer, must-fix still looping to
the reviewer limit, the implementer cap not preempting the reviewer, retryable
blocks bounded then failing closed, a retry that recovers, and a terminal block
failing immediately; concurrency/control/human tests share the new submitRunner
and startRun seams.
Documentation checked: internal/author/skill/references/control.md and
docs/harnesses-and-control.md carry the new edges and the visit-cap rule;
no packaged template defines these pipelines (they are published immutable
versions), and docs/model-results.md needs no change because the {outcome,output}
result contract is unchanged. README and other shared docs deliberately untouched.
Verified: go build ./... and go test -count=1 ./... pass.
Residual limitation: the must-fix-vs-advisory split is carried by the prompt
contract, not a machine-readable marker, because the recorded findings have none;
a model that still returns rejected for advisory-only notes would still buy a
visit. All 18 recorded implementer blocked runs classify terminal, so the
blocked_retryable edge recovers 0 of them.
215 lines
11 KiB
Go
215 lines
11 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// reviewLoopRunner submits a definition through the parser and opens a runner
|
|
// over it, returning the runner, its store, the pipeline id and a per-stage call
|
|
// counter. It reuses the shared submitRunner seam so every assertion below runs
|
|
// the real runLoop, not a copy of its setup or routing.
|
|
func reviewLoopRunner(t *testing.T, ctx context.Context, source string) (*runner, *pipeline.Store, int64, map[string]int) {
|
|
t.Helper()
|
|
r, store, id := submitRunner(t, ctx, source)
|
|
t.Cleanup(func() { store.Close() })
|
|
return r, store, id, map[string]int{}
|
|
}
|
|
|
|
func getRun(t *testing.T, store *pipeline.Store, ctx context.Context, id int64) *protocol.RunDetail {
|
|
t.Helper()
|
|
detail, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return detail
|
|
}
|
|
|
|
// TestReviewerAdvisoryDoesNotRevisitImplementer proves an advisory-only review
|
|
// outcome moves the run forward and never buys another worker visit, while the
|
|
// rejected edge and its limits stay declared. The advisory route is the single
|
|
// definition-level mechanism that keeps a non-blocking rejection from looping.
|
|
func TestReviewerAdvisoryDoesNotRevisitImplementer(t *testing.T) {
|
|
ctx := context.Background()
|
|
r, store, id, calls := reviewLoopRunner(t, ctx, reviewerLoopSource())
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
calls[def.Key]++
|
|
if def.Key == "review" {
|
|
return stage.Result{Outcome: "advisory", Output: stage.Values{"findings": "non-blocking nits only"}}, nil
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil
|
|
}
|
|
run := startRun(t, r, id, "advisory")
|
|
detail := getRun(t, store, ctx, run.ID)
|
|
if calls["worker"] != 1 {
|
|
t.Fatalf("advisory review bought %d worker visits, want 1", calls["worker"])
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 3 {
|
|
t.Fatalf("advisory route ended %s with %d steps, want succeeded with 3", detail.Status, len(detail.Steps))
|
|
}
|
|
if detail.Steps[1].Outcome != "advisory" || detail.Steps[1].Target != "commit" {
|
|
t.Fatalf("advisory step = %+v", detail.Steps[1])
|
|
}
|
|
}
|
|
|
|
// TestReviewerMustFixRejectionLoopsAndStopsAtLimit proves a rejection that names a
|
|
// real defect still routes back to the implementer and still terminates at the
|
|
// reviewer's limits cap with the retained diagnostic, so the rejection path is
|
|
// preserved. It runs the exact published combination - the implementer's own
|
|
// blocked_retryable limits block plus the reviewer's rejected -> implementer edge -
|
|
// rather than the reviewer limits alone.
|
|
func TestReviewerMustFixRejectionLoopsAndStopsAtLimit(t *testing.T) {
|
|
ctx := context.Background()
|
|
r, store, id, calls := reviewLoopRunner(t, ctx, publishedReviewLoopSource(5))
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
calls[def.Key]++
|
|
if def.Key == "reviewer" {
|
|
return stage.Result{Outcome: "rejected", Output: stage.Values{"findings": "unmet acceptance check at worker.go:12"}}, nil
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil
|
|
}
|
|
run := startRun(t, r, id, "mustfix")
|
|
detail := getRun(t, store, ctx, run.ID)
|
|
if calls["reviewer"] != 3 || calls["implementer"] != 3 || len(detail.Steps) != 6 {
|
|
t.Fatalf("must-fix loop = calls %v, %d steps", calls, len(detail.Steps))
|
|
}
|
|
last := detail.Steps[5]
|
|
if detail.Status != "failed" || last.Outcome != "rejected" || last.Control == nil || !last.Control.LimitReached || last.Target != protocol.FailureEnd {
|
|
t.Fatalf("limit did not terminate the must-fix loop: %+v", detail)
|
|
}
|
|
if last.Diagnostic != "Pipelines stopped reviewer after 3 visits on outcome rejected" {
|
|
t.Fatalf("limit diagnostic = %q", last.Diagnostic)
|
|
}
|
|
}
|
|
|
|
// TestImplementerVisitCapMustNotPreemptReviewerLimit pins the regression the
|
|
// published shape had: an implementer limits cap below the reviewer's rejection
|
|
// budget stops the loop on the implementer's pre-execution visit cap, so the
|
|
// reviewer's third visit and its "Pipelines stopped reviewer..." diagnostic are
|
|
// unreachable. The cap must be at least the reviewer's budget (the published
|
|
// pipelines use 5) or the reviewer's rejected control is dead.
|
|
func TestImplementerVisitCapMustNotPreemptReviewerLimit(t *testing.T) {
|
|
ctx := context.Background()
|
|
r, store, id, calls := reviewLoopRunner(t, ctx, publishedReviewLoopSource(2))
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
calls[def.Key]++
|
|
if def.Key == "reviewer" {
|
|
return stage.Result{Outcome: "rejected", Output: stage.Values{"findings": "unmet acceptance check"}}, nil
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil
|
|
}
|
|
run := startRun(t, r, id, "preempted")
|
|
detail := getRun(t, store, ctx, run.ID)
|
|
if calls["reviewer"] != 2 || calls["implementer"] != 2 || len(detail.Steps) != 4 {
|
|
t.Fatalf("implementer cap did not preempt the reviewer: calls %v, %d steps", calls, len(detail.Steps))
|
|
}
|
|
if detail.Status != "failed" || detail.Diagnostic != "stage implementer visit limit reached (2)" {
|
|
t.Fatalf("preempted limit diagnostic = %+v", detail)
|
|
}
|
|
}
|
|
|
|
// TestBlockedRetryableRetriesWithinBudgetThenStops proves a transient block is
|
|
// retried only inside its declared visit bound and then fails closed, so a
|
|
// retryable implementer block cannot become a retry storm.
|
|
func TestBlockedRetryableRetriesWithinBudgetThenStops(t *testing.T) {
|
|
ctx := context.Background()
|
|
r, store, id, calls := reviewLoopRunner(t, ctx, retryableBlockSource(2))
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
calls[def.Key]++
|
|
return stage.Result{Outcome: "blocked_retryable", Output: stage.Values{"blocked_reason": "device lock held"}}, nil
|
|
}
|
|
run := startRun(t, r, id, "retryable")
|
|
detail := getRun(t, store, ctx, run.ID)
|
|
if calls["worker"] != 2 || len(detail.Steps) != 2 {
|
|
t.Fatalf("retry bound = calls %v, %d steps, want 2", calls, len(detail.Steps))
|
|
}
|
|
last := detail.Steps[1]
|
|
if detail.Status != "failed" || last.Outcome != "blocked_retryable" || last.Control == nil || !last.Control.LimitReached || last.Target != protocol.FailureEnd {
|
|
t.Fatalf("retryable block not bounded: %+v", last)
|
|
}
|
|
if last.Diagnostic != "Pipelines stopped worker after 2 visits on outcome blocked_retryable" {
|
|
t.Fatalf("retry bound diagnostic = %q", last.Diagnostic)
|
|
}
|
|
}
|
|
|
|
// TestBlockedRetryableRetryCanRecover proves the bounded retry is useful: a stage
|
|
// that blocks once on a transient cause and succeeds on the retry continues to
|
|
// its normal next stage instead of failing the run.
|
|
func TestBlockedRetryableRetryCanRecover(t *testing.T) {
|
|
ctx := context.Background()
|
|
r, store, id, calls := reviewLoopRunner(t, ctx, retryableBlockSource(3))
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
calls[def.Key]++
|
|
if def.Key == "worker" && calls["worker"] == 1 {
|
|
return stage.Result{Outcome: "blocked_retryable", Output: stage.Values{"blocked_reason": "held device lock"}}, nil
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil
|
|
}
|
|
run := startRun(t, r, id, "recovered")
|
|
detail := getRun(t, store, ctx, run.ID)
|
|
if calls["worker"] != 2 || calls["review"] != 1 || detail.Status != "succeeded" {
|
|
t.Fatalf("retry did not recover: calls %v, status %s", calls, detail.Status)
|
|
}
|
|
}
|
|
|
|
// TestTerminalBlockedFailsImmediately proves a terminal block (an acceptance
|
|
// criterion another attempt cannot change) stops on the first visit with no
|
|
// retry and a meaningful run diagnostic.
|
|
func TestTerminalBlockedFailsImmediately(t *testing.T) {
|
|
ctx := context.Background()
|
|
source := strings.Replace(pipeline.Template([]string{"worker", "review"}, true), "success -> review", "success -> review\nblocked -> #failure", 1)
|
|
r, store, id, calls := reviewLoopRunner(t, ctx, source)
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
calls[def.Key]++
|
|
return stage.Result{Outcome: "blocked", Output: stage.Values{"blocked_reason": "absent capability"}}, nil
|
|
}
|
|
run := startRun(t, r, id, "terminal")
|
|
detail := getRun(t, store, ctx, run.ID)
|
|
if calls["worker"] != 1 || len(detail.Steps) != 1 {
|
|
t.Fatalf("terminal block was retried: calls %v, %d steps", calls, len(detail.Steps))
|
|
}
|
|
if detail.Status != "failed" || detail.Diagnostic != `stage worker returned outcome "blocked"` {
|
|
t.Fatalf("terminal block diagnostic = %+v", detail)
|
|
}
|
|
if detail.Steps[0].Target != protocol.FailureEnd {
|
|
t.Fatalf("terminal block target = %q", detail.Steps[0].Target)
|
|
}
|
|
}
|
|
|
|
// reviewerLoopSource is the worker/review/commit shape the maintenance pipelines
|
|
// use: the reviewer keeps its bounded rejection edge and gains an advisory edge
|
|
// that continues the run without another worker visit.
|
|
func reviewerLoopSource() string {
|
|
source := pipeline.Template([]string{"worker", "review", "commit"}, true)
|
|
source = strings.Replace(source, "# stage review", "# stage review\n\n```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```", 1)
|
|
return strings.Replace(source, "success -> commit", "success -> commit\nadvisory -> commit\nrejected -> worker", 1)
|
|
}
|
|
|
|
// publishedReviewLoopSource reconstructs the published maintenance/quick/focused
|
|
// control shape with the real stage keys: the implementer carries its own
|
|
// blocked_retryable limits block and self-edge, and the reviewer's rejected edge
|
|
// returns to that same implementer. implementerVisits is the implementer's
|
|
// max_visits, which also caps its re-entries from the reviewer, so it must be at
|
|
// least the reviewer's rejection budget.
|
|
func publishedReviewLoopSource(implementerVisits int) string {
|
|
source := pipeline.Template([]string{"implementer", "reviewer", "commit"}, true)
|
|
source = strings.Replace(source, "# stage implementer", "# stage implementer\n\n```limits\n{\"max_visits\":"+strconv.Itoa(implementerVisits)+",\"on\":[\"blocked_retryable\"]}\n```", 1)
|
|
source = strings.Replace(source, "success -> reviewer", "success -> reviewer\nblocked_retryable -> implementer\nblocked -> #failure", 1)
|
|
source = strings.Replace(source, "# stage reviewer", "# stage reviewer\n\n```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```", 1)
|
|
return strings.Replace(source, "success -> commit", "success -> commit\nadvisory -> commit\nrejected -> implementer", 1)
|
|
}
|
|
|
|
// retryableBlockSource is a worker/review shape whose worker declares a bounded
|
|
// blocked_retryable self-edge, mirroring the published definitions.
|
|
func retryableBlockSource(maxVisits int) string {
|
|
source := pipeline.Template([]string{"worker", "review"}, true)
|
|
source = strings.Replace(source, "# stage worker", "# stage worker\n\n```limits\n{\"max_visits\":"+strconv.Itoa(maxVisits)+",\"on\":[\"blocked_retryable\"]}\n```", 1)
|
|
return strings.Replace(source, "success -> review", "success -> review\nblocked_retryable -> worker", 1)
|
|
}
|