package examples_test import ( "fmt" "os" "path/filepath" "slices" "strings" "testing" "git.a-wan.net/alan/pipelines/examples" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) // The graded set, in reading order, with its stage keys. var wantStages = []struct { key string stages []string }{ {"one-stage", []string{"review"}}, {"two-stages", []string{"draft", "check"}}, {"feedback-loop", []string{"implement", "review"}}, {"human-decision", []string{"plan", "decide", "act"}}, } func TestCatalogIsTheGradedExampleSet(t *testing.T) { items := examples.Catalog() if len(items) != len(wantStages) { t.Fatal("unexpected catalog size", len(items)) } for i, item := range items { // Numbered names keep the reading order in name-sorted pipeline lists. if item.Key != wantStages[i].key || !strings.HasPrefix(item.Name, fmt.Sprintf("Example %d: ", i+1)) || item.Description == "" || item.Scheduled { t.Fatal("invalid or out-of-order example metadata", item.Key, item.Name) } graph, err := pipeline.ParseDefinition(item.Source) if err != nil { t.Fatal(item.Key, err) } if graph.Settings != nil { t.Fatal("examples must be manual-only", item.Key) } var keys []string for _, s := range graph.Stages { keys = append(keys, s.Key) if strings.TrimSpace(s.Prompt) == "" || s.FallbackError || s.Edges["error"] != protocol.FailureEnd { t.Fatalf("%s/%s needs a prompt and an explicit error -> #failure edge", item.Key, s.Key) } } if !slices.Equal(keys, wantStages[i].stages) { t.Fatalf("%s stages = %v, want %v", item.Key, keys, wantStages[i].stages) } } } func parsedCatalog(t *testing.T) map[string]protocol.Graph { t.Helper() parsed := map[string]protocol.Graph{} for _, item := range examples.Catalog() { graph, err := pipeline.Parse(item.Source) if err != nil { t.Fatal(item.Key, err) } parsed[item.Key] = graph } return parsed } func stageByKey(t *testing.T, graph protocol.Graph, key string) protocol.Stage { t.Helper() for _, s := range graph.Stages { if s.Key == key { return s } } t.Fatalf("stage %s not found", key) return protocol.Stage{} } // Every model stage declares an explicit role allowlist. Only the stages that // make the requested change may edit; nothing has web tools. var exampleTools = map[string]map[string][]string{ "one-stage": {"review": {"read", "glob", "grep"}}, "two-stages": {"draft": {"read", "glob", "grep"}, "check": {"read", "glob", "grep"}}, "feedback-loop": {"implement": {"read", "glob", "grep", "edit", "shell"}, "review": {"read", "glob", "grep", "shell"}}, "human-decision": {"plan": {"read", "glob", "grep"}, "act": {"read", "glob", "grep", "edit", "shell"}}, } func TestExampleRoleToolRestrictions(t *testing.T) { parsed := parsedCatalog(t) for key, stages := range exampleTools { graph := parsed[key] for stageKey, want := range stages { tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(stageByKey(t, graph, stageKey))) if err != nil || !explicit { t.Fatalf("%s/%s must declare an explicit tools block: %v", key, stageKey, err) } slices.Sort(tools) sortedWant := slices.Clone(want) slices.Sort(sortedWant) if !slices.Equal(tools, sortedWant) { t.Fatalf("%s/%s tools = %v, want %v", key, stageKey, tools, sortedWant) } } } for key, graph := range parsed { for _, s := range graph.Stages { if s.Type == stage.ModelType { if _, ok := exampleTools[key][s.Key]; !ok { t.Fatalf("%s/%s needs a role tool expectation", key, s.Key) } } } } } func TestAllExampleDefinitionsAreRegistered(t *testing.T) { files, err := filepath.Glob("*.md") if err != nil { t.Fatal(err) } registered := map[string]string{} for _, item := range examples.Catalog() { registered[item.Key+".md"] = item.Source } for _, file := range files { if file == "README.md" { continue } data, err := os.ReadFile(file) if err != nil || registered[file] != string(data) { t.Fatalf("%s must ship in the embedded catalog: %v", file, err) } } if len(files)-1 != len(registered) { t.Fatal("every catalog entry needs a checked-in file") } } func TestExamplesBindConfiguredDefaults(t *testing.T) { for _, item := range examples.Catalog() { for _, backend := range []string{"opencode", "pi"} { raw, err := pipeline.Parse(item.Source) if err != nil { t.Fatal(err) } for _, s := range raw.Stages { if s.Type != stage.ModelType { continue } if s.Configs["provider"] != "default" || s.Configs["model"] != "default" || s.Configs["harness"] != "" || s.Configs["delivery"] != "" || len(s.Memory) != 0 || s.Output != "" { t.Fatalf("%s/%s must inherit model and harness without delivery or retired blocks: %v", item.Key, s.Key, s.Configs) } } graph, err := pipeline.ParseDefinition(pipeline.WithDefaults(item.Source, "test-provider/test-model", backend)) if err != nil { t.Fatal(item.Key, err) } for _, s := range graph.Stages { if s.Type != stage.ModelType { continue } if s.Configs["provider"] != "test-provider" || s.Configs["model"] != "test-model" || s.Configs["harness"] != backend { t.Fatalf("%s/%s did not bind configured defaults: %v", item.Key, s.Key, s.Configs) } } } } } // No example commits, pushes or deploys; editing examples leave changes uncommitted. func TestExamplesAreSafe(t *testing.T) { for key, graph := range parsedCatalog(t) { for _, s := range graph.Stages { prompt := strings.ToLower(s.Prompt) edits := slices.Contains(exampleTools[key][s.Key], "edit") if edits && !strings.Contains(prompt, "uncommitted") { t.Fatalf("%s/%s must leave its changes uncommitted", key, s.Key) } if !edits && s.Type == stage.ModelType && !strings.Contains(prompt, "do not edit") && !strings.Contains(prompt, "change nothing") { t.Fatalf("%s/%s must say it changes nothing", key, s.Key) } } } } func TestFeedbackLoopUsesRunnerOwnedLimitsAndRetries(t *testing.T) { graph := parsedCatalog(t)["feedback-loop"] implement, review := stageByKey(t, graph, "implement"), stageByKey(t, graph, "review") if implement.Edges["success"] != "review" || review.Edges["rejected"] != "implement" || review.Edges["approved"] != protocol.SuccessEnd { t.Fatal("feedback loop must route implement -> review, rejected -> implement", implement.Edges, review.Edges) } if review.Limits == nil || review.Limits.MaxVisits != 3 || !slices.Equal(review.Limits.On, []string{"rejected"}) { t.Fatalf("review must cap rejected loops with runner-owned limits: %+v", review.Limits) } if implement.Configs["retries"] != "1" || implement.Configs["timeout"] != "30m" || review.Configs["retries"] != "2" || review.Configs["timeout"] != "15m" { t.Fatal("feedback loop must show explicit retries and timeouts", implement.Configs, review.Configs) } } func TestHumanDecisionWaitsAndNotifies(t *testing.T) { graph := parsedCatalog(t)["human-decision"] plan, decide, act := stageByKey(t, graph, "plan"), stageByKey(t, graph, "decide"), stageByKey(t, graph, "act") if decide.Type != stage.HumanType || decide.Edges["approve"] != "act" || decide.Edges["revise"] != "plan" || decide.Edges["abort"] != protocol.FailureEnd { t.Fatalf("decide must be a human approve/revise/abort stage: %+v", decide) } for _, tc := range []struct { s protocol.Stage want []string }{{plan, []string{"failure"}}, {decide, []string{"waiting"}}, {act, []string{"success", "failure"}}} { if tc.s.Notify == nil || !slices.Equal(tc.s.Notify.On, tc.want) { t.Fatalf("%s notify = %+v, want on=%v", tc.s.Key, tc.s.Notify, tc.want) } } }