Files
pipelines/examples/catalog_test.go
T

192 lines
6.3 KiB
Go

package examples_test
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/examples"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// The shipped starters, in catalog order, with their stage keys.
var wantStages = []struct {
key string
stages []string
}{
{"research", []string{"research", "report"}},
{"task", []string{"Code", "Review", "UIReview", "Land"}},
}
func TestCatalogIsTheGradedExampleSet(t *testing.T) {
items := examples.Catalog()
if len(items) != len(wantStages) {
t.Fatal("unexpected catalog size", len(items))
}
for i, item := range items {
// Numbered names keep the reading order in name-sorted pipeline lists.
if item.Key != wantStages[i].key || item.Name == "" || item.Description == "" || item.Scheduled {
t.Fatal("invalid or out-of-order example metadata", item.Key, item.Name)
}
graph, err := pipeline.ParseDefinition(item.Source)
if err != nil {
t.Fatal(item.Key, err)
}
if graph.Settings != nil {
t.Fatal("examples must be manual-only", item.Key)
}
var keys []string
for _, s := range graph.Stages {
keys = append(keys, s.Key)
if strings.TrimSpace(s.Prompt) == "" || s.FallbackError || s.Edges["error"] == "" {
t.Fatalf("%s/%s needs a prompt and an explicit error -> #failure edge", item.Key, s.Key)
}
}
if !slices.Equal(keys, wantStages[i].stages) {
t.Fatalf("%s stages = %v, want %v", item.Key, keys, wantStages[i].stages)
}
}
}
func parsedCatalog(t *testing.T) map[string]protocol.Graph {
t.Helper()
parsed := map[string]protocol.Graph{}
for _, item := range examples.Catalog() {
graph, err := pipeline.Parse(item.Source)
if err != nil {
t.Fatal(item.Key, err)
}
parsed[item.Key] = graph
}
return parsed
}
func stageByKey(t *testing.T, graph protocol.Graph, key string) protocol.Stage {
t.Helper()
for _, s := range graph.Stages {
if s.Key == key {
return s
}
}
t.Fatalf("stage %s not found", key)
return protocol.Stage{}
}
// Every model stage declares an explicit role allowlist. Only the stages that
// make the requested change may edit; nothing has web tools.
var exampleTools = map[string]map[string][]string{
"research": {"research": {"read", "glob", "grep", "shell", "websearch", "webfetch"}, "report": {"read", "glob", "grep", "edit", "shell"}},
"task": {"Code": {"read", "glob", "grep", "edit", "shell"}, "Review": {"read", "glob", "grep", "shell"}, "UIReview": {"read", "glob", "grep", "shell"}, "Land": {"read", "glob", "grep", "edit", "shell"}},
}
func TestExampleRoleToolRestrictions(t *testing.T) {
parsed := parsedCatalog(t)
for key, stages := range exampleTools {
graph := parsed[key]
for stageKey, want := range stages {
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(stageByKey(t, graph, stageKey)))
if err != nil || !explicit {
t.Fatalf("%s/%s must declare an explicit tools block: %v", key, stageKey, err)
}
slices.Sort(tools)
sortedWant := slices.Clone(want)
slices.Sort(sortedWant)
if !slices.Equal(tools, sortedWant) {
t.Fatalf("%s/%s tools = %v, want %v", key, stageKey, tools, sortedWant)
}
}
}
for key, graph := range parsed {
for _, s := range graph.Stages {
if s.Type == stage.ModelType {
if _, ok := exampleTools[key][s.Key]; !ok {
t.Fatalf("%s/%s needs a role tool expectation", key, s.Key)
}
}
}
}
}
func TestAllExampleDefinitionsAreRegistered(t *testing.T) {
files, err := filepath.Glob("*.md")
if err != nil {
t.Fatal(err)
}
registered := map[string]string{}
for _, item := range examples.Catalog() {
registered[item.Key+".md"] = item.Source
}
for _, file := range files {
if file == "README.md" {
continue
}
data, err := os.ReadFile(file)
if err != nil || registered[file] != string(data) {
t.Fatalf("%s must ship in the embedded catalog: %v", file, err)
}
}
if len(files)-1 != len(registered) {
t.Fatal("every catalog entry needs a checked-in file")
}
}
func TestExamplesBindConfiguredDefaults(t *testing.T) {
for _, item := range examples.Catalog() {
for _, backend := range []string{"opencode", "pi"} {
raw, err := pipeline.Parse(item.Source)
if err != nil {
t.Fatal(err)
}
for _, s := range raw.Stages {
if s.Type != stage.ModelType {
continue
}
if s.Configs["provider"] != "default" || s.Configs["model"] != "default" || s.Configs["harness"] != "" || s.Configs["delivery"] != "" || len(s.Memory) != 0 || s.Output != "" {
t.Fatalf("%s/%s must inherit model and harness without delivery or retired blocks: %v", item.Key, s.Key, s.Configs)
}
}
graph, err := pipeline.ParseDefinition(pipeline.WithDefaults(item.Source, "test-provider/test-model", backend))
if err != nil {
t.Fatal(item.Key, err)
}
for _, s := range graph.Stages {
if s.Type != stage.ModelType {
continue
}
if s.Configs["provider"] != "test-provider" || s.Configs["model"] != "test-model" || s.Configs["harness"] != backend {
t.Fatalf("%s/%s did not bind configured defaults: %v", item.Key, s.Key, s.Configs)
}
}
}
}
}
// Read-only stages state their non-editing boundary; delivery behavior is
// intentionally different between the research and task starters.
func TestExamplesAreSafe(t *testing.T) {
for key, graph := range parsedCatalog(t) {
for _, s := range graph.Stages {
prompt := strings.ToLower(s.Prompt)
edits := slices.Contains(exampleTools[key][s.Key], "edit")
if !edits && s.Type == stage.ModelType && !strings.Contains(prompt, "do not edit") && !strings.Contains(prompt, "change nothing") {
t.Fatalf("%s/%s must say it changes nothing", key, s.Key)
}
}
}
}
func TestTaskUsesRunnerOwnedLimitsAndReviewLoop(t *testing.T) {
graph := parsedCatalog(t)["task"]
code, review, ui := stageByKey(t, graph, "Code"), stageByKey(t, graph, "Review"), stageByKey(t, graph, "UIReview")
if code.Edges["success"] != "Review" || review.Edges["rejected"] != "Code" || ui.Edges["rejected"] != "Code" {
t.Fatal("task must route implementation and review findings through Code", code.Edges, review.Edges, ui.Edges)
}
if review.Limits == nil || review.Limits.MaxVisits != 3 || !slices.Equal(review.Limits.On, []string{"rejected", "error"}) {
t.Fatalf("review must cap rejected/error loops with runner-owned limits: %+v", review.Limits)
}
}