Files
pipelines/internal/server/control_test.go
T
alan e70c94e8dd control: make advisory review findings non-blocking and bound retryable blocks
Cut the review loop's repeat cost and make a blocked stage actionable, at the
definition level only, with the runner's existing visit counting and caps.

The published maintenance, quick and focused pipelines gain a reviewer
`advisory -> next stage` edge for non-blocking notes, so an advisory-only review
continues the run instead of buying another implementer visit; `rejected ->
implementer` and its unchanged {"max_visits":3,"on":["rejected"]} limit keep a
must-fix finding looping and still terminate. The implementer gains
blocked_retryable -> implementer with {"max_visits":5,"on":["blocked_retryable"]}
for a transient environment/tooling/device problem a fresh attempt could
survive, while blocked -> #failure stays terminal. The cap is at least the
reviewer's rejection budget so the implementer's own pre-execution cap cannot
preempt the reviewer's rejected control.

No production Go code changed. New tests in internal/server/review_contract_test.go
cover advisory-only never revisiting the implementer, must-fix still looping to
the reviewer limit, the implementer cap not preempting the reviewer, retryable
blocks bounded then failing closed, a retry that recovers, and a terminal block
failing immediately; concurrency/control/human tests share the new submitRunner
and startRun seams.

Documentation checked: internal/author/skill/references/control.md and
docs/harnesses-and-control.md carry the new edges and the visit-cap rule;
no packaged template defines these pipelines (they are published immutable
versions), and docs/model-results.md needs no change because the {outcome,output}
result contract is unchanged. README and other shared docs deliberately untouched.

Verified: go build ./... and go test -count=1 ./... pass.

Residual limitation: the must-fix-vs-advisory split is carried by the prompt
contract, not a machine-readable marker, because the recorded findings have none;
a model that still returns rejected for advisory-only notes would still buy a
visit. All 18 recorded implementer blocked runs classify terminal, so the
blocked_retryable edge recovers 0 of them.
2026-10-17 17:06:51 +01:00

84 lines
3.4 KiB
Go

package server
import (
"context"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestRuntimeLimitAndSelectedMemory(t *testing.T) {
for _, approveLast := range []bool{false, true} {
t.Run(map[bool]string{false: "exhaustion", true: "last review approves"}[approveLast], func(t *testing.T) {
ctx := context.Background()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "run.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"planner", "worker", "review"}, true)
source = strings.Replace(source, "# stage worker", "# stage worker\n\n```memory\n{\"task\":\"input.prompt\",\"plan\":\"planner.plan\",\"findings\":\"review.findings?\"}\n```", 1)
source = strings.Replace(source, "# stage review", "# stage review\n\n```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```\n\n```memory\n{\"task\":\"input.prompt\",\"checks\":\"planner.checks\"}\n```", 1)
source = strings.Replace(source, "success -> #success", "success -> #success\nrejected -> worker", 1)
if _, err := store.Submit(ctx, "bounded", source); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
runner, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
calls := map[string]int{}
runner.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
calls[def.Key]++
if _, ok := def.Blocks["limits"]; ok {
t.Fatal("policy exposed to model")
}
switch def.Key {
case "planner":
return stage.Result{Outcome: "success", Output: stage.Values{"plan": "do task", "checks": "inspect actual files"}}, nil
case "worker":
if input["task"] != "task" || input["plan"] != "do task" {
t.Error("lost shared task context", input)
}
if calls["worker"] > 1 && input["findings"] != "fix layout" {
t.Error("missing revision feedback", input)
}
return stage.Result{Outcome: "success", Output: stage.Values{"ui_revision": 0, "previous_verdict": "rejected"}}, nil
default:
if len(input) != 2 || input["checks"] != "inspect actual files" {
t.Error("review context contaminated", input)
}
if approveLast && calls["review"] == 3 {
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "approved"}}, nil
}
return stage.Result{Outcome: "rejected", Output: stage.Values{"findings": "fix layout", "ui_revision": 0}}, nil
}
}
run := startRun(t, runner, items[0].ID, "test")
retained, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if calls["review"] != 3 || calls["worker"] != 3 || len(retained.Steps) != 7 {
t.Fatal("model evaded runtime limit", calls, len(retained.Steps))
}
last := retained.Steps[6]
if last.Control == nil || last.Control.Visit != 3 || last.Control.Memory["checks"].Sequence != 1 {
t.Fatal("missing retained control evidence", last)
}
if approveLast {
if retained.Status != "succeeded" || last.Control.LimitReached {
t.Fatal("last visit could not approve", retained)
}
} else if retained.Status != "failed" || !last.Control.LimitReached || last.Outcome != "rejected" || last.Target != protocol.FailureEnd {
t.Fatal("limit did not override route", last)
}
})
}
}