package examples_test import ( "os" "path/filepath" "slices" "strings" "testing" "git.a-wan.net/alan/pipelines/examples" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) func TestReadmeThreeStageExampleIsValid(t *testing.T) { source, err := os.ReadFile("three-stage.md") if err != nil { t.Fatal(err) } graph, err := pipeline.Parse(string(source)) if err != nil { t.Fatal(err) } if len(graph.Stages) != 4 || graph.Settings != nil { t.Fatal("example must be manual-only and four stages") } } func TestCatalogContainsValidUnpublishedTemplates(t *testing.T) { items := examples.Catalog() if len(items) != 8 { t.Fatal("unexpected catalog size", len(items)) } seen := map[string]bool{} wantStages := map[string]int{"task": 2, "review": 1, "research": 2, "three-stage": 4, "debug": 4, "summarize": 2, "compare": 2, "plan": 2} for _, item := range items { if seen[item.Key] || item.Description == "" || item.Name == "" { t.Fatal("invalid template metadata", item.Key) } seen[item.Key] = true graph, err := pipeline.Parse(item.Source) if err != nil { t.Fatal(item.Key, err) } if (graph.Settings != nil) != item.Scheduled { t.Fatal("missing schedule warning", item.Key) } if graph.Settings != nil || len(graph.Stages) > 4 { t.Fatal("starters must be small and manual-only", item.Key) } if len(graph.Stages) != wantStages[item.Key] { t.Fatal("unexpected/non-generic catalog entry", item.Key) } } } // parsedCatalog parses every bundled starter once for the role/route assertions. func parsedCatalog(t *testing.T) map[string]pipeline.Graph { t.Helper() parsed := map[string]pipeline.Graph{} for _, item := range examples.Catalog() { graph, err := pipeline.Parse(item.Source) if err != nil { t.Fatal(item.Key, err) } parsed[item.Key] = graph } return parsed } func stageTools(t *testing.T, graph pipeline.Graph, key string) ([]string, bool) { t.Helper() for _, s := range graph.Stages { if s.Key != key { continue } tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s)) if err != nil { t.Fatalf("stage %s: %v", key, err) } sorted := slices.Clone(tools) slices.Sort(sorted) return sorted, explicit } t.Fatalf("stage %s not found", key) return nil, false } // starterTools is the expected explicit role allowlist per template and stage. // Every bundled starter declares a tools block; an empty list is the intentional // deny-all tool-free role. var starterTools = map[string]map[string][]string{ "task": { "worker": {"read", "glob", "grep", "edit", "shell"}, "review": {"read", "glob", "grep", "shell"}, }, "review": {"review": {"read", "glob", "grep", "websearch", "webfetch"}}, "research": { "research": {"websearch", "webfetch"}, "report": {"webfetch"}, }, "summarize": {"summarize": {"read", "glob", "grep"}, "check": {"read", "glob", "grep"}}, "compare": {"compare": {"read", "glob", "grep"}, "check": {"read", "glob", "grep"}}, "plan": {"plan": {"read", "glob", "grep"}, "check": {"read", "glob", "grep"}}, "three-stage": { "planner": {"read", "glob", "grep", "websearch", "webfetch"}, "implementer": {"read", "glob", "grep", "edit", "shell"}, "reviewer": {"read", "glob", "grep", "shell", "websearch", "webfetch"}, "committer": {"read", "glob", "grep", "shell"}, }, "debug": { "diagnose": {"read", "glob", "grep", "shell"}, "fix": {"read", "glob", "grep", "edit", "shell"}, "verify": {"read", "glob", "grep", "shell"}, "committer": {"read", "glob", "grep", "shell"}, }, } func TestStarterRoleToolRestrictions(t *testing.T) { parsed := parsedCatalog(t) if len(parsed) != len(starterTools) { t.Fatal("every catalog entry must have role tool expectations") } for key, stages := range starterTools { graph, ok := parsed[key] if !ok { t.Fatalf("catalog entry %s missing", key) } if len(stages) != len(graph.Stages) { t.Fatalf("every %s stage must have role tool expectations", key) } for stageKey, want := range stages { tools, explicit := stageTools(t, graph, stageKey) if !explicit { t.Fatalf("%s/%s must declare an explicit tools block", key, stageKey) } sortedWant := slices.Clone(want) slices.Sort(sortedWant) if !slices.Equal(tools, sortedWant) { t.Fatalf("%s/%s tools = %v, want %v", key, stageKey, tools, sortedWant) } } } } func TestAllExampleDefinitionsAreRegistered(t *testing.T) { files, err := filepath.Glob("*.md") if err != nil { t.Fatal(err) } registered := map[string]string{} for _, item := range examples.Catalog() { registered[item.Key+".md"] = item.Source } for _, file := range files { if file == "README.md" { continue } data, err := os.ReadFile(file) if err != nil || registered[file] != string(data) { t.Fatalf("%s must ship in the embedded catalog: %v", file, err) } } } func TestStartersBindConfiguredDefaultsAndOutcomes(t *testing.T) { for _, item := range examples.Catalog() { t.Run(item.Key, func(t *testing.T) { raw, err := pipeline.Parse(item.Source) if err != nil { t.Fatal(err) } for _, s := range raw.Stages { if s.Configs["provider"] != "default" || s.Configs["model"] != "default" || s.Configs["harness"] != "" { t.Fatalf("%s must inherit model and harness, got %v", s.Key, s.Configs) } if s.Type != stage.ModelType || len(s.Memory) != 0 || s.Output != "" || s.Configs["delivery"] != "" { t.Fatalf("%s must use plain model-stage Markdown without delivery or retired blocks", s.Key) } if s.Edges["success"] == "" || (s.Edges["blocked"] != protocol.FailureEnd && s.Edges["rejected"] != protocol.FailureEnd) || s.Edges["error"] != protocol.FailureEnd { t.Fatalf("%s must bind success, blocked/rejected and error outcomes: %v", s.Key, s.Edges) } } bound := pipeline.WithDefaults(item.Source, "test-provider/test-model", "opencode") graph, err := pipeline.Parse(bound) if err != nil { t.Fatal(err) } for _, s := range graph.Stages { if s.Configs["provider"] != "test-provider" || s.Configs["model"] != "test-model" || s.Configs["harness"] != "opencode" { t.Fatalf("%s did not bind configured defaults: %v", s.Key, s.Configs) } } }) } } func TestSimpleLocalStartersSupportConfiguredPiHarness(t *testing.T) { for _, item := range examples.Catalog() { if !slices.Contains([]string{"task", "summarize", "compare", "plan"}, item.Key) { continue } graph, err := pipeline.Parse(pipeline.WithDefaults(item.Source, "test-provider/test-model", "pi")) if err != nil { t.Fatal(item.Key, err) } for _, s := range graph.Stages { if s.Configs["harness"] != "pi" { t.Fatalf("%s/%s ignored configured harness", item.Key, s.Key) } } } } func TestSmallCodeChangeEndsInIndependentReadOnlyReview(t *testing.T) { graph := parsedCatalog(t)["task"] if graph.Stages[0].Edges["success"] != "review" || graph.Stages[1].Edges["success"] != protocol.SuccessEnd { t.Fatal("small code change must route through review") } if !strings.Contains(graph.Stages[1].Prompt, "Independently review the actual changed files") || !strings.Contains(graph.Stages[0].Prompt, "Do not commit, push or deploy") { t.Fatal("small code change must review actual work and leave it uncommitted") } } func TestResearchRequiresSourceEvidenceAndHonestToolLimitations(t *testing.T) { graph := parsedCatalog(t)["research"] if !strings.Contains(graph.Stages[0].Prompt, "If either is unavailable") || !strings.Contains(graph.Stages[1].Prompt, "Fact-check") || !strings.Contains(graph.Stages[1].Prompt, "Never invent citations") { t.Fatal("research must report missing tools and verify, not invent, citations") } } func TestThreeStagePlannerBlockedRoute(t *testing.T) { graph := parsedCatalog(t)["three-stage"] for _, s := range graph.Stages { if s.Key == "planner" { if got := s.Edges["blocked"]; got != protocol.FailureEnd { t.Fatalf("planner blocked route = %q, want %q", got, protocol.FailureEnd) } } } } func TestThreeStageUsesTurnFilesWithoutRetiredBlocks(t *testing.T) { graph := parsedCatalog(t)["three-stage"] for _, s := range graph.Stages { if len(s.Memory) != 0 { t.Fatalf("%s stage %s must not use the retired memory block: %v", "three-stage", s.Key, s.Memory) } if s.Output != "" { t.Fatalf("%s stage %s must not declare the retired output block", "three-stage", s.Key) } } var reviewer protocol.Stage for _, s := range graph.Stages { if s.Key == "reviewer" { reviewer = s } } if reviewer.Key == "" || !strings.Contains(strings.ToLower(reviewer.Prompt), "independently review") { t.Fatalf("three-stage reviewer must independently review the full diff: %+v", reviewer) } } // Existing extended coding starters retain their explicit local-commit behavior. // The small task starter deliberately leaves changes uncommitted for the user. func TestExtendedCodingStartersEndInACommitter(t *testing.T) { for _, tc := range []struct{ key, producer string }{ {"three-stage", "implementer"}, {"debug", "fix"}, } { graph := parsedCatalog(t)[tc.key] last := graph.Stages[len(graph.Stages)-1] if last.Key != "committer" { t.Fatalf("%s must end in a committer stage, got %q", tc.key, last.Key) } if got := last.Edges["success"]; got != protocol.SuccessEnd { t.Fatalf("%s committer success route = %q, want %q", tc.key, got, protocol.SuccessEnd) } if len(last.Memory) != 0 || last.Output != "" { t.Fatalf("%s committer must use the turn files, not retired blocks: %+v", tc.key, last) } if !strings.Contains(last.Prompt, "diff") { t.Fatalf("%s committer prompt must inspect the actual diff", tc.key) } // The producer hands its work on rather than ending the run itself. for _, s := range graph.Stages { if s.Key == tc.producer { if protocol.IsTerminal(s.Edges["success"]) { t.Fatalf("%s/%s must hand success to a later stage, not %q", tc.key, tc.producer, s.Edges["success"]) } } } } }