Files
pipelines/examples/catalog_test.go
T
alan a48e4a5b6a Replace starters with four graded examples seeded into new databases
The template catalog is now Example 1-4: one stage, two stages (handoff),
feedback loop (runner-owned limits, retries, timeout) and human decision
(type=human approve/revise/abort with notify hooks). All read-only or leave
changes uncommitted; none commits, pushes or deploys.

A brand-new database (schema created by this Open) gets the examples once at
server start, bound to the configured default model and harness. A
server_settings marker records the decision, so removed examples never come
back and existing installs are marked without any pipeline being added or
rewritten.
2026-11-03 19:35:47 +00:00

218 lines
7.6 KiB
Go

package examples_test
import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/examples"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// The graded set, in reading order, with its stage keys.
var wantStages = []struct {
key string
stages []string
}{
{"one-stage", []string{"review"}},
{"two-stages", []string{"draft", "check"}},
{"feedback-loop", []string{"implement", "review"}},
{"human-decision", []string{"plan", "decide", "act"}},
}
func TestCatalogIsTheGradedExampleSet(t *testing.T) {
items := examples.Catalog()
if len(items) != len(wantStages) {
t.Fatal("unexpected catalog size", len(items))
}
for i, item := range items {
// Numbered names keep the reading order in name-sorted pipeline lists.
if item.Key != wantStages[i].key || !strings.HasPrefix(item.Name, fmt.Sprintf("Example %d: ", i+1)) || item.Description == "" || item.Scheduled {
t.Fatal("invalid or out-of-order example metadata", item.Key, item.Name)
}
graph, err := pipeline.ParseDefinition(item.Source)
if err != nil {
t.Fatal(item.Key, err)
}
if graph.Settings != nil {
t.Fatal("examples must be manual-only", item.Key)
}
var keys []string
for _, s := range graph.Stages {
keys = append(keys, s.Key)
if strings.TrimSpace(s.Prompt) == "" || s.FallbackError || s.Edges["error"] != protocol.FailureEnd {
t.Fatalf("%s/%s needs a prompt and an explicit error -> #failure edge", item.Key, s.Key)
}
}
if !slices.Equal(keys, wantStages[i].stages) {
t.Fatalf("%s stages = %v, want %v", item.Key, keys, wantStages[i].stages)
}
}
}
func parsedCatalog(t *testing.T) map[string]protocol.Graph {
t.Helper()
parsed := map[string]protocol.Graph{}
for _, item := range examples.Catalog() {
graph, err := pipeline.Parse(item.Source)
if err != nil {
t.Fatal(item.Key, err)
}
parsed[item.Key] = graph
}
return parsed
}
func stageByKey(t *testing.T, graph protocol.Graph, key string) protocol.Stage {
t.Helper()
for _, s := range graph.Stages {
if s.Key == key {
return s
}
}
t.Fatalf("stage %s not found", key)
return protocol.Stage{}
}
// Every model stage declares an explicit role allowlist. Only the stages that
// make the requested change may edit; nothing has web tools.
var exampleTools = map[string]map[string][]string{
"one-stage": {"review": {"read", "glob", "grep"}},
"two-stages": {"draft": {"read", "glob", "grep"}, "check": {"read", "glob", "grep"}},
"feedback-loop": {"implement": {"read", "glob", "grep", "edit", "shell"}, "review": {"read", "glob", "grep", "shell"}},
"human-decision": {"plan": {"read", "glob", "grep"}, "act": {"read", "glob", "grep", "edit", "shell"}},
}
func TestExampleRoleToolRestrictions(t *testing.T) {
parsed := parsedCatalog(t)
for key, stages := range exampleTools {
graph := parsed[key]
for stageKey, want := range stages {
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(stageByKey(t, graph, stageKey)))
if err != nil || !explicit {
t.Fatalf("%s/%s must declare an explicit tools block: %v", key, stageKey, err)
}
slices.Sort(tools)
sortedWant := slices.Clone(want)
slices.Sort(sortedWant)
if !slices.Equal(tools, sortedWant) {
t.Fatalf("%s/%s tools = %v, want %v", key, stageKey, tools, sortedWant)
}
}
}
for key, graph := range parsed {
for _, s := range graph.Stages {
if s.Type == stage.ModelType {
if _, ok := exampleTools[key][s.Key]; !ok {
t.Fatalf("%s/%s needs a role tool expectation", key, s.Key)
}
}
}
}
}
func TestAllExampleDefinitionsAreRegistered(t *testing.T) {
files, err := filepath.Glob("*.md")
if err != nil {
t.Fatal(err)
}
registered := map[string]string{}
for _, item := range examples.Catalog() {
registered[item.Key+".md"] = item.Source
}
for _, file := range files {
if file == "README.md" {
continue
}
data, err := os.ReadFile(file)
if err != nil || registered[file] != string(data) {
t.Fatalf("%s must ship in the embedded catalog: %v", file, err)
}
}
if len(files)-1 != len(registered) {
t.Fatal("every catalog entry needs a checked-in file")
}
}
func TestExamplesBindConfiguredDefaults(t *testing.T) {
for _, item := range examples.Catalog() {
for _, backend := range []string{"opencode", "pi"} {
raw, err := pipeline.Parse(item.Source)
if err != nil {
t.Fatal(err)
}
for _, s := range raw.Stages {
if s.Type != stage.ModelType {
continue
}
if s.Configs["provider"] != "default" || s.Configs["model"] != "default" || s.Configs["harness"] != "" || s.Configs["delivery"] != "" || len(s.Memory) != 0 || s.Output != "" {
t.Fatalf("%s/%s must inherit model and harness without delivery or retired blocks: %v", item.Key, s.Key, s.Configs)
}
}
graph, err := pipeline.ParseDefinition(pipeline.WithDefaults(item.Source, "test-provider/test-model", backend))
if err != nil {
t.Fatal(item.Key, err)
}
for _, s := range graph.Stages {
if s.Type != stage.ModelType {
continue
}
if s.Configs["provider"] != "test-provider" || s.Configs["model"] != "test-model" || s.Configs["harness"] != backend {
t.Fatalf("%s/%s did not bind configured defaults: %v", item.Key, s.Key, s.Configs)
}
}
}
}
}
// No example commits, pushes or deploys; editing examples leave changes uncommitted.
func TestExamplesAreSafe(t *testing.T) {
for key, graph := range parsedCatalog(t) {
for _, s := range graph.Stages {
prompt := strings.ToLower(s.Prompt)
edits := slices.Contains(exampleTools[key][s.Key], "edit")
if edits && !strings.Contains(prompt, "uncommitted") {
t.Fatalf("%s/%s must leave its changes uncommitted", key, s.Key)
}
if !edits && s.Type == stage.ModelType && !strings.Contains(prompt, "do not edit") && !strings.Contains(prompt, "change nothing") {
t.Fatalf("%s/%s must say it changes nothing", key, s.Key)
}
}
}
}
func TestFeedbackLoopUsesRunnerOwnedLimitsAndRetries(t *testing.T) {
graph := parsedCatalog(t)["feedback-loop"]
implement, review := stageByKey(t, graph, "implement"), stageByKey(t, graph, "review")
if implement.Edges["success"] != "review" || review.Edges["rejected"] != "implement" || review.Edges["approved"] != protocol.SuccessEnd {
t.Fatal("feedback loop must route implement -> review, rejected -> implement", implement.Edges, review.Edges)
}
if review.Limits == nil || review.Limits.MaxVisits != 3 || !slices.Equal(review.Limits.On, []string{"rejected"}) {
t.Fatalf("review must cap rejected loops with runner-owned limits: %+v", review.Limits)
}
if implement.Configs["retries"] != "1" || implement.Configs["timeout"] != "30m" || review.Configs["retries"] != "2" || review.Configs["timeout"] != "15m" {
t.Fatal("feedback loop must show explicit retries and timeouts", implement.Configs, review.Configs)
}
}
func TestHumanDecisionWaitsAndNotifies(t *testing.T) {
graph := parsedCatalog(t)["human-decision"]
plan, decide, act := stageByKey(t, graph, "plan"), stageByKey(t, graph, "decide"), stageByKey(t, graph, "act")
if decide.Type != stage.HumanType || decide.Edges["approve"] != "act" || decide.Edges["revise"] != "plan" || decide.Edges["abort"] != protocol.FailureEnd {
t.Fatalf("decide must be a human approve/revise/abort stage: %+v", decide)
}
for _, tc := range []struct {
s protocol.Stage
want []string
}{{plan, []string{"failure"}}, {decide, []string{"waiting"}}, {act, []string{"success", "failure"}}} {
if tc.s.Notify == nil || !slices.Equal(tc.s.Notify.On, tc.want) {
t.Fatalf("%s notify = %+v, want on=%v", tc.s.Key, tc.s.Notify, tc.want)
}
}
}