e70c94e8dd
Cut the review loop's repeat cost and make a blocked stage actionable, at the
definition level only, with the runner's existing visit counting and caps.
The published maintenance, quick and focused pipelines gain a reviewer
`advisory -> next stage` edge for non-blocking notes, so an advisory-only review
continues the run instead of buying another implementer visit; `rejected ->
implementer` and its unchanged {"max_visits":3,"on":["rejected"]} limit keep a
must-fix finding looping and still terminate. The implementer gains
blocked_retryable -> implementer with {"max_visits":5,"on":["blocked_retryable"]}
for a transient environment/tooling/device problem a fresh attempt could
survive, while blocked -> #failure stays terminal. The cap is at least the
reviewer's rejection budget so the implementer's own pre-execution cap cannot
preempt the reviewer's rejected control.
No production Go code changed. New tests in internal/server/review_contract_test.go
cover advisory-only never revisiting the implementer, must-fix still looping to
the reviewer limit, the implementer cap not preempting the reviewer, retryable
blocks bounded then failing closed, a retry that recovers, and a terminal block
failing immediately; concurrency/control/human tests share the new submitRunner
and startRun seams.
Documentation checked: internal/author/skill/references/control.md and
docs/harnesses-and-control.md carry the new edges and the visit-cap rule;
no packaged template defines these pipelines (they are published immutable
versions), and docs/model-results.md needs no change because the {outcome,output}
result contract is unchanged. README and other shared docs deliberately untouched.
Verified: go build ./... and go test -count=1 ./... pass.
Residual limitation: the must-fix-vs-advisory split is carried by the prompt
contract, not a machine-readable marker, because the recorded findings have none;
a model that still returns rejected for advisory-only notes would still buy a
visit. All 18 recorded implementer blocked runs classify terminal, so the
blocked_retryable edge recovers 0 of them.
84 lines
3.4 KiB
Go
84 lines
3.4 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
func TestRuntimeLimitAndSelectedMemory(t *testing.T) {
|
|
for _, approveLast := range []bool{false, true} {
|
|
t.Run(map[bool]string{false: "exhaustion", true: "last review approves"}[approveLast], func(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "run.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
source := pipeline.Template([]string{"planner", "worker", "review"}, true)
|
|
source = strings.Replace(source, "# stage worker", "# stage worker\n\n```memory\n{\"task\":\"input.prompt\",\"plan\":\"planner.plan\",\"findings\":\"review.findings?\"}\n```", 1)
|
|
source = strings.Replace(source, "# stage review", "# stage review\n\n```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```\n\n```memory\n{\"task\":\"input.prompt\",\"checks\":\"planner.checks\"}\n```", 1)
|
|
source = strings.Replace(source, "success -> #success", "success -> #success\nrejected -> worker", 1)
|
|
if _, err := store.Submit(ctx, "bounded", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
runner, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
calls := map[string]int{}
|
|
runner.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
|
calls[def.Key]++
|
|
if _, ok := def.Blocks["limits"]; ok {
|
|
t.Fatal("policy exposed to model")
|
|
}
|
|
switch def.Key {
|
|
case "planner":
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"plan": "do task", "checks": "inspect actual files"}}, nil
|
|
case "worker":
|
|
if input["task"] != "task" || input["plan"] != "do task" {
|
|
t.Error("lost shared task context", input)
|
|
}
|
|
if calls["worker"] > 1 && input["findings"] != "fix layout" {
|
|
t.Error("missing revision feedback", input)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"ui_revision": 0, "previous_verdict": "rejected"}}, nil
|
|
default:
|
|
if len(input) != 2 || input["checks"] != "inspect actual files" {
|
|
t.Error("review context contaminated", input)
|
|
}
|
|
if approveLast && calls["review"] == 3 {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "approved"}}, nil
|
|
}
|
|
return stage.Result{Outcome: "rejected", Output: stage.Values{"findings": "fix layout", "ui_revision": 0}}, nil
|
|
}
|
|
}
|
|
run := startRun(t, runner, items[0].ID, "test")
|
|
retained, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if calls["review"] != 3 || calls["worker"] != 3 || len(retained.Steps) != 7 {
|
|
t.Fatal("model evaded runtime limit", calls, len(retained.Steps))
|
|
}
|
|
last := retained.Steps[6]
|
|
if last.Control == nil || last.Control.Visit != 3 || last.Control.Memory["checks"].Sequence != 1 {
|
|
t.Fatal("missing retained control evidence", last)
|
|
}
|
|
if approveLast {
|
|
if retained.Status != "succeeded" || last.Control.LimitReached {
|
|
t.Fatal("last visit could not approve", retained)
|
|
}
|
|
} else if retained.Status != "failed" || !last.Control.LimitReached || last.Outcome != "rejected" || last.Target != protocol.FailureEnd {
|
|
t.Fatal("limit did not override route", last)
|
|
}
|
|
})
|
|
}
|
|
}
|