Files
pipelines/examples/catalog_test.go
T

299 lines
9.9 KiB
Go

package examples_test
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/examples"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestReadmeThreeStageExampleIsValid(t *testing.T) {
source, err := os.ReadFile("three-stage.md")
if err != nil {
t.Fatal(err)
}
graph, err := pipeline.Parse(string(source))
if err != nil {
t.Fatal(err)
}
if len(graph.Stages) != 4 || graph.Settings != nil {
t.Fatal("example must be manual-only and four stages")
}
}
func TestCatalogContainsValidUnpublishedTemplates(t *testing.T) {
items := examples.Catalog()
if len(items) != 8 {
t.Fatal("unexpected catalog size", len(items))
}
seen := map[string]bool{}
wantStages := map[string]int{"task": 2, "review": 1, "research": 2, "three-stage": 4, "debug": 4, "summarize": 2, "compare": 2, "plan": 2}
for _, item := range items {
if seen[item.Key] || item.Description == "" || item.Name == "" {
t.Fatal("invalid template metadata", item.Key)
}
seen[item.Key] = true
graph, err := pipeline.Parse(item.Source)
if err != nil {
t.Fatal(item.Key, err)
}
if (graph.Settings != nil) != item.Scheduled {
t.Fatal("missing schedule warning", item.Key)
}
if graph.Settings != nil || len(graph.Stages) > 4 {
t.Fatal("starters must be small and manual-only", item.Key)
}
if len(graph.Stages) != wantStages[item.Key] {
t.Fatal("unexpected/non-generic catalog entry", item.Key)
}
}
}
// parsedCatalog parses every bundled starter once for the role/route assertions.
func parsedCatalog(t *testing.T) map[string]pipeline.Graph {
t.Helper()
parsed := map[string]pipeline.Graph{}
for _, item := range examples.Catalog() {
graph, err := pipeline.Parse(item.Source)
if err != nil {
t.Fatal(item.Key, err)
}
parsed[item.Key] = graph
}
return parsed
}
func stageTools(t *testing.T, graph pipeline.Graph, key string) ([]string, bool) {
t.Helper()
for _, s := range graph.Stages {
if s.Key != key {
continue
}
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
if err != nil {
t.Fatalf("stage %s: %v", key, err)
}
sorted := slices.Clone(tools)
slices.Sort(sorted)
return sorted, explicit
}
t.Fatalf("stage %s not found", key)
return nil, false
}
// starterTools is the expected explicit role allowlist per template and stage.
// Every bundled starter declares a tools block; an empty list is the intentional
// deny-all tool-free role.
var starterTools = map[string]map[string][]string{
"task": {
"worker": {"read", "glob", "grep", "edit", "shell"},
"review": {"read", "glob", "grep", "shell"},
},
"review": {"review": {"read", "glob", "grep", "websearch", "webfetch"}},
"research": {
"research": {"websearch", "webfetch"},
"report": {"webfetch"},
},
"summarize": {"summarize": {"read", "glob", "grep"}, "check": {"read", "glob", "grep"}},
"compare": {"compare": {"read", "glob", "grep"}, "check": {"read", "glob", "grep"}},
"plan": {"plan": {"read", "glob", "grep"}, "check": {"read", "glob", "grep"}},
"three-stage": {
"planner": {"read", "glob", "grep", "websearch", "webfetch"},
"implementer": {"read", "glob", "grep", "edit", "shell"},
"reviewer": {"read", "glob", "grep", "shell", "websearch", "webfetch"},
"committer": {"read", "glob", "grep", "shell"},
},
"debug": {
"diagnose": {"read", "glob", "grep", "shell"},
"fix": {"read", "glob", "grep", "edit", "shell"},
"verify": {"read", "glob", "grep", "shell"},
"committer": {"read", "glob", "grep", "shell"},
},
}
func TestStarterRoleToolRestrictions(t *testing.T) {
parsed := parsedCatalog(t)
if len(parsed) != len(starterTools) {
t.Fatal("every catalog entry must have role tool expectations")
}
for key, stages := range starterTools {
graph, ok := parsed[key]
if !ok {
t.Fatalf("catalog entry %s missing", key)
}
if len(stages) != len(graph.Stages) {
t.Fatalf("every %s stage must have role tool expectations", key)
}
for stageKey, want := range stages {
tools, explicit := stageTools(t, graph, stageKey)
if !explicit {
t.Fatalf("%s/%s must declare an explicit tools block", key, stageKey)
}
sortedWant := slices.Clone(want)
slices.Sort(sortedWant)
if !slices.Equal(tools, sortedWant) {
t.Fatalf("%s/%s tools = %v, want %v", key, stageKey, tools, sortedWant)
}
}
}
}
func TestAllExampleDefinitionsAreRegistered(t *testing.T) {
files, err := filepath.Glob("*.md")
if err != nil {
t.Fatal(err)
}
registered := map[string]string{}
for _, item := range examples.Catalog() {
registered[item.Key+".md"] = item.Source
}
for _, file := range files {
if file == "README.md" {
continue
}
data, err := os.ReadFile(file)
if err != nil || registered[file] != string(data) {
t.Fatalf("%s must ship in the embedded catalog: %v", file, err)
}
}
}
func TestStartersBindConfiguredDefaultsAndOutcomes(t *testing.T) {
for _, item := range examples.Catalog() {
t.Run(item.Key, func(t *testing.T) {
raw, err := pipeline.Parse(item.Source)
if err != nil {
t.Fatal(err)
}
for _, s := range raw.Stages {
if s.Configs["provider"] != "default" || s.Configs["model"] != "default" || s.Configs["harness"] != "" {
t.Fatalf("%s must inherit model and harness, got %v", s.Key, s.Configs)
}
if s.Type != stage.ModelType || len(s.Memory) != 0 || s.Output != "" || s.Configs["delivery"] != "" {
t.Fatalf("%s must use plain model-stage Markdown without delivery or retired blocks", s.Key)
}
if s.Edges["success"] == "" || (s.Edges["blocked"] != protocol.FailureEnd && s.Edges["rejected"] != protocol.FailureEnd) || s.Edges["error"] != protocol.FailureEnd {
t.Fatalf("%s must bind success, blocked/rejected and error outcomes: %v", s.Key, s.Edges)
}
}
bound := pipeline.WithDefaults(item.Source, "test-provider/test-model", "opencode")
graph, err := pipeline.Parse(bound)
if err != nil {
t.Fatal(err)
}
for _, s := range graph.Stages {
if s.Configs["provider"] != "test-provider" || s.Configs["model"] != "test-model" || s.Configs["harness"] != "opencode" {
t.Fatalf("%s did not bind configured defaults: %v", s.Key, s.Configs)
}
}
})
}
}
func TestSimpleLocalStartersSupportConfiguredPiHarness(t *testing.T) {
for _, item := range examples.Catalog() {
if !slices.Contains([]string{"task", "summarize", "compare", "plan"}, item.Key) {
continue
}
graph, err := pipeline.Parse(pipeline.WithDefaults(item.Source, "test-provider/test-model", "pi"))
if err != nil {
t.Fatal(item.Key, err)
}
for _, s := range graph.Stages {
if s.Configs["harness"] != "pi" {
t.Fatalf("%s/%s ignored configured harness", item.Key, s.Key)
}
}
}
}
func TestSmallCodeChangeEndsInIndependentReadOnlyReview(t *testing.T) {
graph := parsedCatalog(t)["task"]
if graph.Stages[0].Edges["success"] != "review" || graph.Stages[1].Edges["success"] != protocol.SuccessEnd {
t.Fatal("small code change must route through review")
}
if !strings.Contains(graph.Stages[1].Prompt, "Independently review the actual changed files") ||
!strings.Contains(graph.Stages[0].Prompt, "Do not commit, push or deploy") {
t.Fatal("small code change must review actual work and leave it uncommitted")
}
}
func TestResearchRequiresSourceEvidenceAndHonestToolLimitations(t *testing.T) {
graph := parsedCatalog(t)["research"]
if !strings.Contains(graph.Stages[0].Prompt, "If either is unavailable") ||
!strings.Contains(graph.Stages[1].Prompt, "Fact-check") ||
!strings.Contains(graph.Stages[1].Prompt, "Never invent citations") {
t.Fatal("research must report missing tools and verify, not invent, citations")
}
}
func TestThreeStagePlannerBlockedRoute(t *testing.T) {
graph := parsedCatalog(t)["three-stage"]
for _, s := range graph.Stages {
if s.Key == "planner" {
if got := s.Edges["blocked"]; got != protocol.FailureEnd {
t.Fatalf("planner blocked route = %q, want %q", got, protocol.FailureEnd)
}
}
}
}
func TestThreeStageUsesTurnFilesWithoutRetiredBlocks(t *testing.T) {
graph := parsedCatalog(t)["three-stage"]
for _, s := range graph.Stages {
if len(s.Memory) != 0 {
t.Fatalf("%s stage %s must not use the retired memory block: %v", "three-stage", s.Key, s.Memory)
}
if s.Output != "" {
t.Fatalf("%s stage %s must not declare the retired output block", "three-stage", s.Key)
}
}
var reviewer protocol.Stage
for _, s := range graph.Stages {
if s.Key == "reviewer" {
reviewer = s
}
}
if reviewer.Key == "" || !strings.Contains(strings.ToLower(reviewer.Prompt), "independently review") {
t.Fatalf("three-stage reviewer must independently review the full diff: %+v", reviewer)
}
}
// Existing extended coding starters retain their explicit local-commit behavior.
// The small task starter deliberately leaves changes uncommitted for the user.
func TestExtendedCodingStartersEndInACommitter(t *testing.T) {
for _, tc := range []struct{ key, producer string }{
{"three-stage", "implementer"},
{"debug", "fix"},
} {
graph := parsedCatalog(t)[tc.key]
last := graph.Stages[len(graph.Stages)-1]
if last.Key != "committer" {
t.Fatalf("%s must end in a committer stage, got %q", tc.key, last.Key)
}
if got := last.Edges["success"]; got != protocol.SuccessEnd {
t.Fatalf("%s committer success route = %q, want %q", tc.key, got, protocol.SuccessEnd)
}
if len(last.Memory) != 0 || last.Output != "" {
t.Fatalf("%s committer must use the turn files, not retired blocks: %+v", tc.key, last)
}
if !strings.Contains(last.Prompt, "diff") {
t.Fatalf("%s committer prompt must inspect the actual diff", tc.key)
}
// The producer hands its work on rather than ending the run itself.
for _, s := range graph.Stages {
if s.Key == tc.producer {
if protocol.IsTerminal(s.Edges["success"]) {
t.Fatalf("%s/%s must hand success to a later stage, not %q", tc.key, tc.producer, s.Edges["success"])
}
}
}
}
}