192 lines
6.3 KiB
Go
192 lines
6.3 KiB
Go
package examples_test
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/examples"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// The shipped starters, in catalog order, with their stage keys.
|
|
var wantStages = []struct {
|
|
key string
|
|
stages []string
|
|
}{
|
|
{"research", []string{"research", "report"}},
|
|
{"task", []string{"Code", "Review", "UIReview", "Land"}},
|
|
}
|
|
|
|
func TestCatalogIsTheGradedExampleSet(t *testing.T) {
|
|
items := examples.Catalog()
|
|
if len(items) != len(wantStages) {
|
|
t.Fatal("unexpected catalog size", len(items))
|
|
}
|
|
for i, item := range items {
|
|
// Numbered names keep the reading order in name-sorted pipeline lists.
|
|
if item.Key != wantStages[i].key || item.Name == "" || item.Description == "" || item.Scheduled {
|
|
t.Fatal("invalid or out-of-order example metadata", item.Key, item.Name)
|
|
}
|
|
graph, err := pipeline.ParseDefinition(item.Source)
|
|
if err != nil {
|
|
t.Fatal(item.Key, err)
|
|
}
|
|
if graph.Settings != nil {
|
|
t.Fatal("examples must be manual-only", item.Key)
|
|
}
|
|
var keys []string
|
|
for _, s := range graph.Stages {
|
|
keys = append(keys, s.Key)
|
|
if strings.TrimSpace(s.Prompt) == "" || s.FallbackError || s.Edges["error"] == "" {
|
|
t.Fatalf("%s/%s needs a prompt and an explicit error -> #failure edge", item.Key, s.Key)
|
|
}
|
|
}
|
|
if !slices.Equal(keys, wantStages[i].stages) {
|
|
t.Fatalf("%s stages = %v, want %v", item.Key, keys, wantStages[i].stages)
|
|
}
|
|
}
|
|
}
|
|
|
|
func parsedCatalog(t *testing.T) map[string]protocol.Graph {
|
|
t.Helper()
|
|
parsed := map[string]protocol.Graph{}
|
|
for _, item := range examples.Catalog() {
|
|
graph, err := pipeline.Parse(item.Source)
|
|
if err != nil {
|
|
t.Fatal(item.Key, err)
|
|
}
|
|
parsed[item.Key] = graph
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
func stageByKey(t *testing.T, graph protocol.Graph, key string) protocol.Stage {
|
|
t.Helper()
|
|
for _, s := range graph.Stages {
|
|
if s.Key == key {
|
|
return s
|
|
}
|
|
}
|
|
t.Fatalf("stage %s not found", key)
|
|
return protocol.Stage{}
|
|
}
|
|
|
|
// Every model stage declares an explicit role allowlist. Only the stages that
|
|
// make the requested change may edit; nothing has web tools.
|
|
var exampleTools = map[string]map[string][]string{
|
|
"research": {"research": {"read", "glob", "grep", "shell", "websearch", "webfetch"}, "report": {"read", "glob", "grep", "edit", "shell"}},
|
|
"task": {"Code": {"read", "glob", "grep", "edit", "shell"}, "Review": {"read", "glob", "grep", "shell"}, "UIReview": {"read", "glob", "grep", "shell"}, "Land": {"read", "glob", "grep", "edit", "shell"}},
|
|
}
|
|
|
|
func TestExampleRoleToolRestrictions(t *testing.T) {
|
|
parsed := parsedCatalog(t)
|
|
for key, stages := range exampleTools {
|
|
graph := parsed[key]
|
|
for stageKey, want := range stages {
|
|
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(stageByKey(t, graph, stageKey)))
|
|
if err != nil || !explicit {
|
|
t.Fatalf("%s/%s must declare an explicit tools block: %v", key, stageKey, err)
|
|
}
|
|
slices.Sort(tools)
|
|
sortedWant := slices.Clone(want)
|
|
slices.Sort(sortedWant)
|
|
if !slices.Equal(tools, sortedWant) {
|
|
t.Fatalf("%s/%s tools = %v, want %v", key, stageKey, tools, sortedWant)
|
|
}
|
|
}
|
|
}
|
|
for key, graph := range parsed {
|
|
for _, s := range graph.Stages {
|
|
if s.Type == stage.ModelType {
|
|
if _, ok := exampleTools[key][s.Key]; !ok {
|
|
t.Fatalf("%s/%s needs a role tool expectation", key, s.Key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAllExampleDefinitionsAreRegistered(t *testing.T) {
|
|
files, err := filepath.Glob("*.md")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
registered := map[string]string{}
|
|
for _, item := range examples.Catalog() {
|
|
registered[item.Key+".md"] = item.Source
|
|
}
|
|
for _, file := range files {
|
|
if file == "README.md" {
|
|
continue
|
|
}
|
|
data, err := os.ReadFile(file)
|
|
if err != nil || registered[file] != string(data) {
|
|
t.Fatalf("%s must ship in the embedded catalog: %v", file, err)
|
|
}
|
|
}
|
|
if len(files)-1 != len(registered) {
|
|
t.Fatal("every catalog entry needs a checked-in file")
|
|
}
|
|
}
|
|
|
|
func TestExamplesBindConfiguredDefaults(t *testing.T) {
|
|
for _, item := range examples.Catalog() {
|
|
for _, backend := range []string{"opencode", "pi"} {
|
|
raw, err := pipeline.Parse(item.Source)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, s := range raw.Stages {
|
|
if s.Type != stage.ModelType {
|
|
continue
|
|
}
|
|
if s.Configs["provider"] != "default" || s.Configs["model"] != "default" || s.Configs["harness"] != "" || s.Configs["delivery"] != "" || len(s.Memory) != 0 || s.Output != "" {
|
|
t.Fatalf("%s/%s must inherit model and harness without delivery or retired blocks: %v", item.Key, s.Key, s.Configs)
|
|
}
|
|
}
|
|
graph, err := pipeline.ParseDefinition(pipeline.WithDefaults(item.Source, "test-provider/test-model", backend))
|
|
if err != nil {
|
|
t.Fatal(item.Key, err)
|
|
}
|
|
for _, s := range graph.Stages {
|
|
if s.Type != stage.ModelType {
|
|
continue
|
|
}
|
|
if s.Configs["provider"] != "test-provider" || s.Configs["model"] != "test-model" || s.Configs["harness"] != backend {
|
|
t.Fatalf("%s/%s did not bind configured defaults: %v", item.Key, s.Key, s.Configs)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Read-only stages state their non-editing boundary; delivery behavior is
|
|
// intentionally different between the research and task starters.
|
|
func TestExamplesAreSafe(t *testing.T) {
|
|
for key, graph := range parsedCatalog(t) {
|
|
for _, s := range graph.Stages {
|
|
prompt := strings.ToLower(s.Prompt)
|
|
edits := slices.Contains(exampleTools[key][s.Key], "edit")
|
|
if !edits && s.Type == stage.ModelType && !strings.Contains(prompt, "do not edit") && !strings.Contains(prompt, "change nothing") {
|
|
t.Fatalf("%s/%s must say it changes nothing", key, s.Key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTaskUsesRunnerOwnedLimitsAndReviewLoop(t *testing.T) {
|
|
graph := parsedCatalog(t)["task"]
|
|
code, review, ui := stageByKey(t, graph, "Code"), stageByKey(t, graph, "Review"), stageByKey(t, graph, "UIReview")
|
|
if code.Edges["success"] != "Review" || review.Edges["rejected"] != "Code" || ui.Edges["rejected"] != "Code" {
|
|
t.Fatal("task must route implementation and review findings through Code", code.Edges, review.Edges, ui.Edges)
|
|
}
|
|
if review.Limits == nil || review.Limits.MaxVisits != 3 || !slices.Equal(review.Limits.On, []string{"rejected", "error"}) {
|
|
t.Fatalf("review must cap rejected/error loops with runner-owned limits: %+v", review.Limits)
|
|
}
|
|
}
|